junafinity's picture
claude opus 5.5
VeriSafe-Tanglish-1-STT (STT-2)
ec01e2d
Raw History Blame Contribute Delete
7.14 kB
"""Text normalization and the CTC tokenizers. Index 0 is always the CTC blank.
- CTCTokenizer: the subset of Gemma tokenizer pieces seen in training (QUAN-Transcribe used ~15k).
Fine for one language (Hindi: 5.5k pieces), but Tamil + English gives 30.7k pieces.
- SPMTokenizer: a compact SentencePiece BPE trained on the transcripts. CTC loss cost grows ~linearly
with vocabulary (measured per batch: 31 ms at 1k, 107 ms at 5.5k, 678 ms at 30.7k), and eg2-tamil
found 1,024 Tamil BPE units beat Gemma tokens. Use it for multilingual / Tamil runs.
"""
from __future__ import annotations
import json
import re
import unicodedata
from collections import Counter
from pathlib import Path
from transformers import AutoTokenizer
from .model import BASE_MODEL
# Danda, double danda, ASCII and common Unicode punctuation. Hyphens become spaces; apostrophes
# inside words (don't, it's) survive via _INNER_APOS, all others become spaces.
_PUNCT = re.compile(r"[।॥!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~‘’“”–—… ]")
_SPACE = re.compile(r"\s+")
_INNER_APOS = re.compile(r"(?<=\w)['’](?=\w)")
_KEEP = "" # private-use placeholder for a protected apostrophe
_NUKTA = "़"
# Precomposed nukta letters -> base letter (क़ ख़ ग़ ज़ ड़ ढ़ फ़ य़)
_NUKTA_FOLD = str.maketrans({"क़": "क", "ख़": "ख", "ग़": "ग", "ज़": "ज",
"ड़": "ड", "ढ़": "ढ", "फ़": "फ", "य़": "य"})
def normalize(text: str, fold_nukta: bool = False) -> str:
"""Training target / scoring normalization: NFC, no punctuation, single spaces.
fold_nukta=True also maps nukta letters to their base letter (क़ -> क). Use it for a
"lenient" WER that ignores the nukta spelling variation common in Hindi transcripts.
"""
text = unicodedata.normalize("NFC", text).lower() # lower() only changes cased scripts (English)
text = _INNER_APOS.sub(_KEEP, text)
text = _PUNCT.sub(" ", text).replace(_KEEP, "'")
if fold_nukta:
text = unicodedata.normalize("NFD", text).replace(_NUKTA, "")
text = unicodedata.normalize("NFC", text).translate(_NUKTA_FOLD)
return _SPACE.sub(" ", text).strip()
class CTCTokenizer:
def __init__(self, gemma_ids: list[int]):
self.gemma = AutoTokenizer.from_pretrained(BASE_MODEL)
self.gemma_ids = list(gemma_ids)
self.to_ctc = {g: i + 1 for i, g in enumerate(self.gemma_ids)}
@property
def vocab_size(self) -> int:
return len(self.gemma_ids) + 1 # + blank
@classmethod
def build(cls, texts, min_count: int = 1) -> "CTCTokenizer":
tok = AutoTokenizer.from_pretrained(BASE_MODEL)
counts = Counter()
for t in texts:
counts.update(tok.encode(normalize(t), add_special_tokens=False))
ids = sorted(g for g, c in counts.items() if c >= min_count)
return cls(ids)
def save(self, path: str | Path) -> None:
Path(path).write_text(json.dumps({"base": BASE_MODEL, "gemma_ids": self.gemma_ids}))
@classmethod
def load(cls, path: str | Path) -> "CTCTokenizer":
return cls(json.loads(Path(path).read_text())["gemma_ids"])
def encode(self, text: str) -> list[int] | None:
"""CTC targets for a transcript, or None if it contains a piece outside the vocabulary."""
out = []
for g in self.gemma.encode(normalize(text), add_special_tokens=False):
c = self.to_ctc.get(g)
if c is None:
return None
out.append(c)
return out
def decode(self, ctc_ids: list[int]) -> str:
gemma = [self.gemma_ids[i - 1] for i in ctc_ids if i > 0]
return self.gemma.decode(gemma, skip_special_tokens=True).strip()
def greedy(self, log_probs, lengths) -> list[str]:
"""Greedy CTC decoding: argmax per frame, merge repeats, drop blanks."""
best = log_probs.argmax(-1).cpu()
out = []
for row, n in zip(best, lengths.tolist()):
ids, prev = [], -1
for i in row[:n].tolist():
if i != prev and i != 0:
ids.append(i)
prev = i
out.append(self.decode(ids))
return out
class SPMTokenizer:
def __init__(self, model_path: str | Path):
import sentencepiece as spm
self.path = str(model_path)
self.sp = spm.SentencePieceProcessor(model_file=self.path)
@property
def vocab_size(self) -> int:
return self.sp.get_piece_size() + 1 # CTC id = SentencePiece id + 1 (0 = blank)
@classmethod
def build(cls, texts, path: str | Path, vocab_size: int = 2048) -> "SPMTokenizer":
import tempfile
import sentencepiece as spm
with tempfile.NamedTemporaryFile("w", suffix=".txt", encoding="utf-8", delete=False) as f:
for t in texts:
n = normalize(t)
if n:
f.write(n + "\n")
spm.SentencePieceTrainer.train(
input=f.name, model_prefix=str(path).removesuffix(".model"), vocab_size=vocab_size,
model_type="bpe", character_coverage=1.0, unk_id=0, bos_id=-1, eos_id=-1, pad_id=-1,
input_sentence_size=2_000_000, shuffle_input_sentence=True, num_threads=16, minloglevel=2,
)
Path(f.name).unlink(missing_ok=True)
return cls(path)
def encode(self, text: str) -> list[int] | None:
"""CTC targets, or None if the transcript is empty or has a character outside the vocabulary."""
ids = self.sp.encode(normalize(text))
if not ids or 0 in ids: # 0 = <unk>
return None
return [i + 1 for i in ids]
def decode(self, ctc_ids: list[int]) -> str:
return self.sp.decode([i - 1 for i in ctc_ids if i > 0]).strip()
greedy = CTCTokenizer.greedy
def script_ids(self) -> dict[str, list[int]]:
"""Model ids (0 = blank, id = SentencePiece id + 1) of the pieces with a Latin letter and with a Tamil character.
Blank, digits, the apostrophe, the bare word marker and <unk> are in neither list, so they stay allowed under any
language hint. Same classes as rnnt_beam.script_masks.
"""
out = {"latin": [], "tamil": []}
for i in range(self.sp.get_piece_size()):
if self.sp.is_unknown(i): # its name has letters, but it is not a Latin piece
continue
piece = self.sp.id_to_piece(i)
latin = any(c.isalpha() and unicodedata.name(c, "").startswith("LATIN") for c in piece)
tamil = any("஀" <= c <= "௿" for c in piece)
if latin and tamil:
raise ValueError(f"piece {i} has both scripts: decide its class before masking")
if latin:
out["latin"].append(i + 1)
if tamil:
out["tamil"].append(i + 1)
return out
def load_tokenizer(path: str | Path) -> CTCTokenizer | SPMTokenizer:
"""spm.model -> SPMTokenizer, vocab.json -> CTCTokenizer."""
return SPMTokenizer(path) if str(path).endswith(".model") else CTCTokenizer.load(path)