"""Text side: the Omnilingual char vocabulary, normalisation, CTC targets and greedy decoding. ``omniASR_tokenizer`` is a character-level SentencePiece model (9,812 entries: Latin, Khmer and ~1,600 other languages' characters). It is case-sensitive with lowercase entries only, maps a space to its own token, and has no entry for Khmer punctuation (``។``, ``៕``) or zero-width characters, which it turns into ````. Omnilingual's own training pipeline drops ```` from targets, so this module does the same. The vocabulary is read once from fairseq2 and kept as a plain ``{char: id}`` table: a dict lookup per character is exactly what the char model does, and it is safe to use in forked DataLoader workers. """ from __future__ import annotations import json import re import unicodedata from pathlib import Path BLANK = 0 # fairseq2's ctc_loss uses the default blank, index 0 () SPECIAL = {0, 1, 2, 3} # , , , UNK = 3 SPACE_PIECE = "▁" _ZERO_WIDTH = re.compile("[​‌‍⁠]") _SPACES = re.compile(r"\s+") def normalize_text(text: str, lang: str) -> str: """NFC, zero-width characters removed, whitespace collapsed; English lowercased.""" text = unicodedata.normalize("NFC", text) text = _ZERO_WIDTH.sub("", text) if lang == "en": text = text.lower() return _SPACES.sub(" ", text).strip() class CharVocab: def __init__(self, pieces: list[str]) -> None: self.pieces = pieces self.index = {} for i, p in enumerate(pieces): if i in SPECIAL: continue self.index.setdefault(" " if p == SPACE_PIECE else p, i) @classmethod def from_fairseq2(cls, card: str) -> CharVocab: import torch from fairseq2.data.tokenizers import load_tokenizer tok = load_tokenizer(card) dec = tok.create_decoder() n = tok.vocab_info.size pieces = [] for i in range(n): s = dec(torch.tensor([i])) # the decoder renders the space piece as "" (a bare word boundary) pieces.append(SPACE_PIECE if s == "" and i not in SPECIAL else s) return cls(pieces) def save(self, path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(self.pieces, ensure_ascii=False), encoding="utf-8") @classmethod def load(cls, path: Path) -> CharVocab: return cls(json.loads(path.read_text(encoding="utf-8"))) def __len__(self) -> int: return len(self.pieces) def encode(self, text: str) -> tuple[list[int], int]: """Token ids for normalised ``text`` and the number of characters dropped as unknown.""" ids, unk = [], 0 for ch in text: i = self.index.get(ch) if i is None: unk += 1 else: ids.append(i) # dropping a character can leave double or edge spaces out: list[int] = [] sp = self.index[" "] for i in ids: if i == sp and (not out or out[-1] == sp): continue out.append(i) while out and out[-1] == sp: out.pop() return out, unk def filter(self, text: str) -> str: """``text`` with the characters the model cannot emit removed (reference side of CER).""" return self.decode(self.encode(text)[0]) def decode(self, ids: list[int]) -> str: return "".join(" " if self.pieces[i] == SPACE_PIECE else self.pieces[i] for i in ids if i not in SPECIAL) def greedy(self, frame_ids: list[int]) -> str: """CTC greedy decode of one utterance's per-frame argmax ids.""" out, prev = [], None for i in frame_ids: if i != prev and i not in SPECIAL: out.append(i) prev = i return self.decode(out)