"""Text normalization and the CTC tokenizers. Index 0 is always the CTC blank. - CTCTokenizer: the subset of Gemma tokenizer pieces seen in training (QUAN-Transcribe used ~15k). Fine for one language (Hindi: 5.5k pieces), but Tamil + English gives 30.7k pieces. - SPMTokenizer: a compact SentencePiece BPE trained on the transcripts. CTC loss cost grows ~linearly with vocabulary (measured per batch: 31 ms at 1k, 107 ms at 5.5k, 678 ms at 30.7k), and eg2-tamil found 1,024 Tamil BPE units beat Gemma tokens. Use it for multilingual / Tamil runs. """ from __future__ import annotations import json import re import unicodedata from collections import Counter from pathlib import Path from transformers import AutoTokenizer from .model import BASE_MODEL # Danda, double danda, ASCII and common Unicode punctuation. Hyphens become spaces; apostrophes # inside words (don't, it's) survive via _INNER_APOS, all others become spaces. _PUNCT = re.compile(r"[।॥!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~‘’“”–—… ]") _SPACE = re.compile(r"\s+") _INNER_APOS = re.compile(r"(?<=\w)['’](?=\w)") _KEEP = "" # private-use placeholder for a protected apostrophe _NUKTA = "़" # Precomposed nukta letters -> base letter (क़ ख़ ग़ ज़ ड़ ढ़ फ़ य़) _NUKTA_FOLD = str.maketrans({"क़": "क", "ख़": "ख", "ग़": "ग", "ज़": "ज", "ड़": "ड", "ढ़": "ढ", "फ़": "फ", "य़": "य"}) def normalize(text: str, fold_nukta: bool = False) -> str: """Training target / scoring normalization: NFC, no punctuation, single spaces. fold_nukta=True also maps nukta letters to their base letter (क़ -> क). Use it for a "lenient" WER that ignores the nukta spelling variation common in Hindi transcripts. """ text = unicodedata.normalize("NFC", text).lower() # lower() only changes cased scripts (English) text = _INNER_APOS.sub(_KEEP, text) text = _PUNCT.sub(" ", text).replace(_KEEP, "'") if fold_nukta: text = unicodedata.normalize("NFD", text).replace(_NUKTA, "") text = unicodedata.normalize("NFC", text).translate(_NUKTA_FOLD) return _SPACE.sub(" ", text).strip() class CTCTokenizer: def __init__(self, gemma_ids: list[int]): self.gemma = AutoTokenizer.from_pretrained(BASE_MODEL) self.gemma_ids = list(gemma_ids) self.to_ctc = {g: i + 1 for i, g in enumerate(self.gemma_ids)} @property def vocab_size(self) -> int: return len(self.gemma_ids) + 1 # + blank @classmethod def build(cls, texts, min_count: int = 1) -> "CTCTokenizer": tok = AutoTokenizer.from_pretrained(BASE_MODEL) counts = Counter() for t in texts: counts.update(tok.encode(normalize(t), add_special_tokens=False)) ids = sorted(g for g, c in counts.items() if c >= min_count) return cls(ids) def save(self, path: str | Path) -> None: Path(path).write_text(json.dumps({"base": BASE_MODEL, "gemma_ids": self.gemma_ids})) @classmethod def load(cls, path: str | Path) -> "CTCTokenizer": return cls(json.loads(Path(path).read_text())["gemma_ids"]) def encode(self, text: str) -> list[int] | None: """CTC targets for a transcript, or None if it contains a piece outside the vocabulary.""" out = [] for g in self.gemma.encode(normalize(text), add_special_tokens=False): c = self.to_ctc.get(g) if c is None: return None out.append(c) return out def decode(self, ctc_ids: list[int]) -> str: gemma = [self.gemma_ids[i - 1] for i in ctc_ids if i > 0] return self.gemma.decode(gemma, skip_special_tokens=True).strip() def greedy(self, log_probs, lengths) -> list[str]: """Greedy CTC decoding: argmax per frame, merge repeats, drop blanks.""" best = log_probs.argmax(-1).cpu() out = [] for row, n in zip(best, lengths.tolist()): ids, prev = [], -1 for i in row[:n].tolist(): if i != prev and i != 0: ids.append(i) prev = i out.append(self.decode(ids)) return out class SPMTokenizer: def __init__(self, model_path: str | Path): import sentencepiece as spm self.path = str(model_path) self.sp = spm.SentencePieceProcessor(model_file=self.path) @property def vocab_size(self) -> int: return self.sp.get_piece_size() + 1 # CTC id = SentencePiece id + 1 (0 = blank) @classmethod def build(cls, texts, path: str | Path, vocab_size: int = 2048) -> "SPMTokenizer": import tempfile import sentencepiece as spm with tempfile.NamedTemporaryFile("w", suffix=".txt", encoding="utf-8", delete=False) as f: for t in texts: n = normalize(t) if n: f.write(n + "\n") spm.SentencePieceTrainer.train( input=f.name, model_prefix=str(path).removesuffix(".model"), vocab_size=vocab_size, model_type="bpe", character_coverage=1.0, unk_id=0, bos_id=-1, eos_id=-1, pad_id=-1, input_sentence_size=2_000_000, shuffle_input_sentence=True, num_threads=16, minloglevel=2, ) Path(f.name).unlink(missing_ok=True) return cls(path) def encode(self, text: str) -> list[int] | None: """CTC targets, or None if the transcript is empty or has a character outside the vocabulary.""" ids = self.sp.encode(normalize(text)) if not ids or 0 in ids: # 0 = return None return [i + 1 for i in ids] def decode(self, ctc_ids: list[int]) -> str: return self.sp.decode([i - 1 for i in ctc_ids if i > 0]).strip() greedy = CTCTokenizer.greedy def script_ids(self) -> dict[str, list[int]]: """Model ids (0 = blank, id = SentencePiece id + 1) of the pieces with a Latin letter and with a Tamil character. Blank, digits, the apostrophe, the bare word marker and are in neither list, so they stay allowed under any language hint. Same classes as rnnt_beam.script_masks. """ out = {"latin": [], "tamil": []} for i in range(self.sp.get_piece_size()): if self.sp.is_unknown(i): # its name has letters, but it is not a Latin piece continue piece = self.sp.id_to_piece(i) latin = any(c.isalpha() and unicodedata.name(c, "").startswith("LATIN") for c in piece) tamil = any("஀" <= c <= "௿" for c in piece) if latin and tamil: raise ValueError(f"piece {i} has both scripts: decide its class before masking") if latin: out["latin"].append(i + 1) if tamil: out["tamil"].append(i + 1) return out def load_tokenizer(path: str | Path) -> CTCTokenizer | SPMTokenizer: """spm.model -> SPMTokenizer, vocab.json -> CTCTokenizer.""" return SPMTokenizer(path) if str(path).endswith(".model") else CTCTokenizer.load(path)