Automatic Speech Recognition
PyTorch
Safetensors
GGUF
MLX
Tamil
English
speech
asr
streaming
rnnt
ctc
tamil
tanglish
code-switching
indian-english
verisafe
Instructions to use junafinity/VeriSafe-Tanglish-1-STT with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use junafinity/VeriSafe-Tanglish-1-STT with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download junafinity/VeriSafe-Tanglish-1-STT --local-dir VeriSafe-Tanglish-1-STT
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download code/indic_stt/text.py from junafinity/VeriSafe-Tanglish-1-STT: direct link, hf CLI and curl.
- Browser
- Download file 7.14 kB
-
https://huggingface.co/junafinity/VeriSafe-Tanglish-1-STT/resolve/main/code/indic_stt/text.py
- Command line
-
hf download hf://junafinity/VeriSafe-Tanglish-1-STT/code/indic_stt/text.py
-
curl -L -o text.py https://huggingface.co/junafinity/VeriSafe-Tanglish-1-STT/resolve/main/code/indic_stt/text.py
7.14 kB
| """Text normalization and the CTC tokenizers. Index 0 is always the CTC blank. | |
| - CTCTokenizer: the subset of Gemma tokenizer pieces seen in training (QUAN-Transcribe used ~15k). | |
| Fine for one language (Hindi: 5.5k pieces), but Tamil + English gives 30.7k pieces. | |
| - SPMTokenizer: a compact SentencePiece BPE trained on the transcripts. CTC loss cost grows ~linearly | |
| with vocabulary (measured per batch: 31 ms at 1k, 107 ms at 5.5k, 678 ms at 30.7k), and eg2-tamil | |
| found 1,024 Tamil BPE units beat Gemma tokens. Use it for multilingual / Tamil runs. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import re | |
| import unicodedata | |
| from collections import Counter | |
| from pathlib import Path | |
| from transformers import AutoTokenizer | |
| from .model import BASE_MODEL | |
| # Danda, double danda, ASCII and common Unicode punctuation. Hyphens become spaces; apostrophes | |
| # inside words (don't, it's) survive via _INNER_APOS, all others become spaces. | |
| _PUNCT = re.compile(r"[।॥!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~‘’“”–—… ]") | |
| _SPACE = re.compile(r"\s+") | |
| _INNER_APOS = re.compile(r"(?<=\w)['’](?=\w)") | |
| _KEEP = "" # private-use placeholder for a protected apostrophe | |
| _NUKTA = "़" | |
| # Precomposed nukta letters -> base letter (क़ ख़ ग़ ज़ ड़ ढ़ फ़ य़) | |
| _NUKTA_FOLD = str.maketrans({"क़": "क", "ख़": "ख", "ग़": "ग", "ज़": "ज", | |
| "ड़": "ड", "ढ़": "ढ", "फ़": "फ", "य़": "य"}) | |
| def normalize(text: str, fold_nukta: bool = False) -> str: | |
| """Training target / scoring normalization: NFC, no punctuation, single spaces. | |
| fold_nukta=True also maps nukta letters to their base letter (क़ -> क). Use it for a | |
| "lenient" WER that ignores the nukta spelling variation common in Hindi transcripts. | |
| """ | |
| text = unicodedata.normalize("NFC", text).lower() # lower() only changes cased scripts (English) | |
| text = _INNER_APOS.sub(_KEEP, text) | |
| text = _PUNCT.sub(" ", text).replace(_KEEP, "'") | |
| if fold_nukta: | |
| text = unicodedata.normalize("NFD", text).replace(_NUKTA, "") | |
| text = unicodedata.normalize("NFC", text).translate(_NUKTA_FOLD) | |
| return _SPACE.sub(" ", text).strip() | |
| class CTCTokenizer: | |
| def __init__(self, gemma_ids: list[int]): | |
| self.gemma = AutoTokenizer.from_pretrained(BASE_MODEL) | |
| self.gemma_ids = list(gemma_ids) | |
| self.to_ctc = {g: i + 1 for i, g in enumerate(self.gemma_ids)} | |
| def vocab_size(self) -> int: | |
| return len(self.gemma_ids) + 1 # + blank | |
| def build(cls, texts, min_count: int = 1) -> "CTCTokenizer": | |
| tok = AutoTokenizer.from_pretrained(BASE_MODEL) | |
| counts = Counter() | |
| for t in texts: | |
| counts.update(tok.encode(normalize(t), add_special_tokens=False)) | |
| ids = sorted(g for g, c in counts.items() if c >= min_count) | |
| return cls(ids) | |
| def save(self, path: str | Path) -> None: | |
| Path(path).write_text(json.dumps({"base": BASE_MODEL, "gemma_ids": self.gemma_ids})) | |
| def load(cls, path: str | Path) -> "CTCTokenizer": | |
| return cls(json.loads(Path(path).read_text())["gemma_ids"]) | |
| def encode(self, text: str) -> list[int] | None: | |
| """CTC targets for a transcript, or None if it contains a piece outside the vocabulary.""" | |
| out = [] | |
| for g in self.gemma.encode(normalize(text), add_special_tokens=False): | |
| c = self.to_ctc.get(g) | |
| if c is None: | |
| return None | |
| out.append(c) | |
| return out | |
| def decode(self, ctc_ids: list[int]) -> str: | |
| gemma = [self.gemma_ids[i - 1] for i in ctc_ids if i > 0] | |
| return self.gemma.decode(gemma, skip_special_tokens=True).strip() | |
| def greedy(self, log_probs, lengths) -> list[str]: | |
| """Greedy CTC decoding: argmax per frame, merge repeats, drop blanks.""" | |
| best = log_probs.argmax(-1).cpu() | |
| out = [] | |
| for row, n in zip(best, lengths.tolist()): | |
| ids, prev = [], -1 | |
| for i in row[:n].tolist(): | |
| if i != prev and i != 0: | |
| ids.append(i) | |
| prev = i | |
| out.append(self.decode(ids)) | |
| return out | |
| class SPMTokenizer: | |
| def __init__(self, model_path: str | Path): | |
| import sentencepiece as spm | |
| self.path = str(model_path) | |
| self.sp = spm.SentencePieceProcessor(model_file=self.path) | |
| def vocab_size(self) -> int: | |
| return self.sp.get_piece_size() + 1 # CTC id = SentencePiece id + 1 (0 = blank) | |
| def build(cls, texts, path: str | Path, vocab_size: int = 2048) -> "SPMTokenizer": | |
| import tempfile | |
| import sentencepiece as spm | |
| with tempfile.NamedTemporaryFile("w", suffix=".txt", encoding="utf-8", delete=False) as f: | |
| for t in texts: | |
| n = normalize(t) | |
| if n: | |
| f.write(n + "\n") | |
| spm.SentencePieceTrainer.train( | |
| input=f.name, model_prefix=str(path).removesuffix(".model"), vocab_size=vocab_size, | |
| model_type="bpe", character_coverage=1.0, unk_id=0, bos_id=-1, eos_id=-1, pad_id=-1, | |
| input_sentence_size=2_000_000, shuffle_input_sentence=True, num_threads=16, minloglevel=2, | |
| ) | |
| Path(f.name).unlink(missing_ok=True) | |
| return cls(path) | |
| def encode(self, text: str) -> list[int] | None: | |
| """CTC targets, or None if the transcript is empty or has a character outside the vocabulary.""" | |
| ids = self.sp.encode(normalize(text)) | |
| if not ids or 0 in ids: # 0 = <unk> | |
| return None | |
| return [i + 1 for i in ids] | |
| def decode(self, ctc_ids: list[int]) -> str: | |
| return self.sp.decode([i - 1 for i in ctc_ids if i > 0]).strip() | |
| greedy = CTCTokenizer.greedy | |
| def script_ids(self) -> dict[str, list[int]]: | |
| """Model ids (0 = blank, id = SentencePiece id + 1) of the pieces with a Latin letter and with a Tamil character. | |
| Blank, digits, the apostrophe, the bare word marker and <unk> are in neither list, so they stay allowed under any | |
| language hint. Same classes as rnnt_beam.script_masks. | |
| """ | |
| out = {"latin": [], "tamil": []} | |
| for i in range(self.sp.get_piece_size()): | |
| if self.sp.is_unknown(i): # its name has letters, but it is not a Latin piece | |
| continue | |
| piece = self.sp.id_to_piece(i) | |
| latin = any(c.isalpha() and unicodedata.name(c, "").startswith("LATIN") for c in piece) | |
| tamil = any("" <= c <= "" for c in piece) | |
| if latin and tamil: | |
| raise ValueError(f"piece {i} has both scripts: decide its class before masking") | |
| if latin: | |
| out["latin"].append(i + 1) | |
| if tamil: | |
| out["tamil"].append(i + 1) | |
| return out | |
| def load_tokenizer(path: str | Path) -> CTCTokenizer | SPMTokenizer: | |
| """spm.model -> SPMTokenizer, vocab.json -> CTCTokenizer.""" | |
| return SPMTokenizer(path) if str(path).endswith(".model") else CTCTokenizer.load(path) | |