File size: 3,881 Bytes
e2525c9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | """Text side: the Omnilingual char vocabulary, normalisation, CTC targets and greedy decoding.
``omniASR_tokenizer`` is a character-level SentencePiece model (9,812 entries: Latin,
Khmer and ~1,600 other languages' characters). It is case-sensitive with lowercase
entries only, maps a space to its own token, and has no entry for Khmer punctuation
(``។``, ``៕``) or zero-width characters, which it turns into ``<unk>``. Omnilingual's
own training pipeline drops ``<unk>`` from targets, so this module does the same.
The vocabulary is read once from fairseq2 and kept as a plain ``{char: id}`` table:
a dict lookup per character is exactly what the char model does, and it is safe to
use in forked DataLoader workers.
"""
from __future__ import annotations
import json
import re
import unicodedata
from pathlib import Path
BLANK = 0 # fairseq2's ctc_loss uses the default blank, index 0 (<s>)
SPECIAL = {0, 1, 2, 3} # <s>, <pad>, </s>, <unk>
UNK = 3
SPACE_PIECE = "▁"
_ZERO_WIDTH = re.compile("[]")
_SPACES = re.compile(r"\s+")
def normalize_text(text: str, lang: str) -> str:
"""NFC, zero-width characters removed, whitespace collapsed; English lowercased."""
text = unicodedata.normalize("NFC", text)
text = _ZERO_WIDTH.sub("", text)
if lang == "en":
text = text.lower()
return _SPACES.sub(" ", text).strip()
class CharVocab:
def __init__(self, pieces: list[str]) -> None:
self.pieces = pieces
self.index = {}
for i, p in enumerate(pieces):
if i in SPECIAL:
continue
self.index.setdefault(" " if p == SPACE_PIECE else p, i)
@classmethod
def from_fairseq2(cls, card: str) -> CharVocab:
import torch
from fairseq2.data.tokenizers import load_tokenizer
tok = load_tokenizer(card)
dec = tok.create_decoder()
n = tok.vocab_info.size
pieces = []
for i in range(n):
s = dec(torch.tensor([i]))
# the decoder renders the space piece as "" (a bare word boundary)
pieces.append(SPACE_PIECE if s == "" and i not in SPECIAL else s)
return cls(pieces)
def save(self, path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(self.pieces, ensure_ascii=False), encoding="utf-8")
@classmethod
def load(cls, path: Path) -> CharVocab:
return cls(json.loads(path.read_text(encoding="utf-8")))
def __len__(self) -> int:
return len(self.pieces)
def encode(self, text: str) -> tuple[list[int], int]:
"""Token ids for normalised ``text`` and the number of characters dropped as unknown."""
ids, unk = [], 0
for ch in text:
i = self.index.get(ch)
if i is None:
unk += 1
else:
ids.append(i)
# dropping a character can leave double or edge spaces
out: list[int] = []
sp = self.index[" "]
for i in ids:
if i == sp and (not out or out[-1] == sp):
continue
out.append(i)
while out and out[-1] == sp:
out.pop()
return out, unk
def filter(self, text: str) -> str:
"""``text`` with the characters the model cannot emit removed (reference side of CER)."""
return self.decode(self.encode(text)[0])
def decode(self, ids: list[int]) -> str:
return "".join(" " if self.pieces[i] == SPACE_PIECE else self.pieces[i] for i in ids if i not in SPECIAL)
def greedy(self, frame_ids: list[int]) -> str:
"""CTC greedy decode of one utterance's per-frame argmax ids."""
out, prev = [], None
for i in frame_ids:
if i != prev and i not in SPECIAL:
out.append(i)
prev = i
return self.decode(out)
|