Automatic Speech Recognition
PyTorch
Safetensors
GGUF
MLX
Tamil
English
speech
asr
streaming
rnnt
ctc
tamil
tanglish
code-switching
indian-english
verisafe
Instructions to use junafinity/VeriSafe-Tanglish-1-STT with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use junafinity/VeriSafe-Tanglish-1-STT with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download junafinity/VeriSafe-Tanglish-1-STT --local-dir VeriSafe-Tanglish-1-STT
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download code/ctc_assist.py from junafinity/VeriSafe-Tanglish-1-STT: direct link, hf CLI and curl.
- Browser
- Download file 10.6 kB
-
https://huggingface.co/junafinity/VeriSafe-Tanglish-1-STT/resolve/main/code/ctc_assist.py
- Command line
-
hf download hf://junafinity/VeriSafe-Tanglish-1-STT/code/ctc_assist.py
-
curl -L -o ctc_assist.py https://huggingface.co/junafinity/VeriSafe-Tanglish-1-STT/resolve/main/code/ctc_assist.py
10.6 kB
| """--ctc-assist for transcribe.py: the J(l, b) CTC-assisted rescoring of the project's scripts/decode_ctc_assist.py, offline. | |
| Candidates: the RNN-T head's beam-4 n-best (modified beam search, no LM, no start penalty) and the CTC head's greedy | |
| hypothesis, duplicates merged. Each gets (1 - l) log P_RNNT(y) + l log P_CTC(y) + b |y| (|y| in tokens; both log P are exact | |
| forced scores summed over all alignments), and the best wins; ties keep the earlier candidate (beam order, then CTC). The | |
| release decodes with l = 0.3, b = 0 (transcribe.py LAMBDA, BETA), as `decode_ctc_assist.py --joint 0.3:0` ran. | |
| The functions below are verbatim copies (cut by source segment, not retyped) of the project's | |
| scripts/rnnt_beam.py sha256 5f83a87194f17e02478fd212d58a1588bf5a7f0a4f76f1b5d93276696e321766: Hyp, log_add, beam_decode | |
| scripts/decode_ctc_assist.py sha256 0d63a090992df8fd8e78d36be548306696adba727d6dd1c0a7baefb2bc8cfae1: JOINT_CELLS, ctc_forced, rnnt_forced, ctc_greedy | |
| joint_rescore() is decode_ctc_assist.decode_clip's J branch for one (l, b). The LM options of beam_decode are unused here. | |
| """ | |
| from __future__ import annotations | |
| import heapq | |
| import math | |
| import torch | |
| class Hyp: | |
| __slots__ = ("ys", "score", "pred", "state", "lm_state", "lm_next") | |
| def __init__(self, ys, score, pred, state, lm_state): | |
| self.ys, self.score, self.pred, self.state, self.lm_state = ys, score, pred, state, lm_state | |
| self.lm_next = {} # token -> (ln P, next LM state); a hypothesis stays in the beam over many blank frames | |
| def lm_step(self, lm: TokenLM, token: int): | |
| res = self.lm_next.get(token) | |
| if res is None: | |
| res = self.lm_next[token] = lm.step(self.lm_state, token) | |
| return res | |
| def log_add(a: float, b: float) -> float: | |
| return (a + math.log1p(math.exp(b - a))) if a >= b else (b + math.log1p(math.exp(a - b))) | |
| JOINT_CELLS = 8192 # T x (U+1) cells of one joint slice in rnnt_forced: 8192 x V x (4 + 8) bytes, about 200 MB at V = 2059 | |
| def ctc_forced(logp: torch.Tensor, y: list[int]) -> float: | |
| """log P_CTC(y | x), summed over all alignments: the CTC forward recursion in float64 on (T, V) log-probs, blank 0. | |
| -inf if y cannot fit in T frames. Equals -F.ctc_loss(reduction="sum") on the same log-probs.""" | |
| T = logp.shape[0] | |
| if not y: | |
| return float(logp[:, 0].double().sum()) | |
| ext = torch.zeros(2 * len(y) + 1, dtype=torch.long) | |
| ext[1::2] = torch.tensor(y) | |
| lp = logp[:, ext].double() # (T, S) | |
| skip = torch.zeros(len(ext), dtype=torch.bool) # s-2 -> s: a token state whose token differs from the previous one | |
| skip[3::2] = ext[3::2] != ext[1:-2:2] | |
| ninf = torch.tensor([-math.inf], dtype=torch.float64) | |
| alpha = torch.full((len(ext),), -math.inf, dtype=torch.float64) | |
| alpha[:2] = lp[0, :2] | |
| for t in range(1, T): | |
| a1 = torch.cat([ninf, alpha[:-1]]) | |
| a2 = torch.where(skip, torch.cat([ninf, ninf, alpha[:-2]]), ninf) | |
| alpha = torch.logsumexp(torch.stack([alpha, a1, a2]), 0) + lp[t] | |
| return float(torch.logaddexp(alpha[-1], alpha[-2])) | |
| def rnnt_forced(head: Transducer, proj: torch.Tensor, y: list[int], style: int | None = None, cells: int = JOINT_CELLS) -> float: | |
| """log P_RNNT(y | x), summed over all alignments of the standard transducer lattice (the training loss's; any number of | |
| tokens per frame). proj = head.enc_proj(enc), (T, J). The joint is built a few frames at a time (<= cells of T x (U+1)) and | |
| only the blank and next-token log-probs are kept, so memory stays small. The log-softmax and the recursion (over u a | |
| log-cumsum-exp per frame) are float64 on the head's float32 logits: this equals a plain float64 lattice DP (measured | |
| <= 1e-6); torchaudio's float32 rnnt_loss is off by up to about 1e-3 absolute (measured on SPRING valid clips).""" | |
| T, U = proj.shape[0], len(y) | |
| tokens = torch.tensor([[0] + y]) | |
| pred, _ = head.predict(tokens) if style is None else head.predict(tokens, None, torch.full((1,), style)) | |
| pred = pred[0] # (U+1, J) | |
| blank = torch.empty(T, U + 1, dtype=torch.float64) | |
| emit = torch.empty(T, U, dtype=torch.float64) | |
| idx = torch.tensor(y)[None, :, None] | |
| step = max(1, cells // (U + 1)) | |
| for t0 in range(0, T, step): | |
| lg = head.joint(proj[t0 : t0 + step, None], pred[None]).double().log_softmax(-1) # (t, U+1, V) | |
| blank[t0 : t0 + step] = lg[..., 0] | |
| if U: | |
| emit[t0 : t0 + step] = lg[:, :U].gather(-1, idx.expand(lg.shape[0], U, 1))[..., 0] | |
| zero = torch.zeros(1, dtype=torch.float64) | |
| alpha = torch.full((U + 1,), -math.inf, dtype=torch.float64) | |
| alpha[0] = 0.0 | |
| for t in range(T): | |
| a = alpha if t == 0 else alpha + blank[t - 1] # from (t-1, u) by blank | |
| c = torch.cat([zero, emit[t].cumsum(0)]) # alpha(t, u) = c[u] + log sum_{k<=u} exp(a[k] - c[k]): tokens within frame t | |
| alpha = c + torch.logcumsumexp(a - c, 0) | |
| return float(alpha[U] + blank[T - 1, U]) | |
| def ctc_greedy(best: torch.Tensor) -> tuple[list[int], list[int]]: | |
| """Frame argmax (T,) -> tokens and each token's first frame: merge repeats, drop blanks (CTCTokenizer.greedy's rule).""" | |
| toks, frames, prev = [], [], -1 | |
| for t, k in enumerate(best.tolist()): | |
| if k != prev and k != 0: | |
| toks.append(k) | |
| frames.append(t) | |
| prev = k | |
| return toks, frames | |
| def beam_decode(head, enc: torch.Tensor, lm: TokenLM | None = None, alpha: float = 0.0, beta: float = 0.0, beam: int = 4, | |
| topk: int = 8, prune: float = 12.0, blank_scoring: bool = False, bigram=None, gamma: float = 0.0, | |
| banned: torch.Tensor | None = None, swap: bool = False, first_bonus: float = 0.0, | |
| style: int | None = None) -> list[tuple[float, tuple[int, ...]]]: | |
| """Modified beam search over one clip's reader output (T, D) -> all final hypotheses, best first, as (score, tokens). | |
| Each frame extends each hypothesis by blank (same tokens, prediction and LM state, no bonus) or by one of its topk | |
| tokens, skipping tokens more than `prune` below that hypothesis' best RNN-T log-prob. The top `beam` survive after | |
| merging. Because the LM and bonus terms depend only on the tokens, merged paths add their RNN-T terms alone. | |
| Options (module docstring): blank_scoring needs the LM; bigram (V, V) of ln P, row 0 = sentence start; banned (V,) bool. | |
| """ | |
| def predict(tokens, states=None): | |
| return head.predict(tokens, states) if style is None else head.predict(tokens, states, torch.full((len(tokens),), style)) | |
| fuse = lm is not None and alpha != 0 | |
| blank_scale = 1.0 + alpha if fuse and blank_scoring else 1.0 | |
| lodr = gamma if bigram is not None else 0.0 | |
| proj = head.enc_proj(enc.float()) | |
| pred, state = predict(torch.zeros(1, 1, dtype=torch.long)) # start symbol = blank, as in training | |
| hyps = [Hyp((), 0.0, pred[0, 0], state, lm.begin() if fuse else None)] | |
| for frame in proj: | |
| logp = head.joint(frame[None], torch.stack([h.pred for h in hyps])).log_softmax(-1) | |
| must_emit = [False] * len(hyps) | |
| if banned is None: | |
| values, tokens = logp[:, 1:].topk(min(topk, logp.shape[1] - 1), dim=-1) | |
| else: | |
| if swap: | |
| must_emit = banned[logp.argmax(-1)].tolist() | |
| values, tokens = logp[:, 1:].masked_fill(banned[1:], -math.inf).topk(min(topk, logp.shape[1] - 1), dim=-1) | |
| blanks = logp[:, 0].tolist() | |
| cands = {h.ys: [h.score + blank_scale * b, h, None] for h, b, m in zip(hyps, blanks, must_emit) if not m} | |
| for h, b, m, vs, ks in zip(hyps, blanks, must_emit, values.tolist(), tokens.tolist()): | |
| floor = (vs[0] if m else max(b, vs[0])) - prune | |
| token_term = alpha * math.log(max(1.0 - math.exp(b), 1e-6)) if blank_scale != 1.0 else 0.0 | |
| for v, k in zip(vs, ks): | |
| if v < floor or v == -math.inf: # a banned token is -inf even with no pruning | |
| break # sorted | |
| k += 1 | |
| score = h.score + v + beta + token_term | |
| if first_bonus and not h.ys: | |
| score += first_bonus | |
| if fuse: | |
| score += alpha * h.lm_step(lm, k)[0] | |
| if lodr: | |
| score -= lodr * float(bigram[h.ys[-1] if h.ys else 0, k]) | |
| ys = h.ys + (k,) | |
| seen = cands.get(ys) | |
| if seen is None: | |
| cands[ys] = [score, h, k] | |
| else: # an existing hypothesis that took blank here: same tokens, so same LM, bonus and LODR terms | |
| seen[0] = log_add(seen[0], score) | |
| top = heapq.nlargest(beam, cands.items(), key=lambda kv: kv[1][0]) | |
| emitted = [c for _, c in top if c[2] is not None] | |
| if emitted: | |
| y, (hn, cn) = predict(torch.tensor([[c[2]] for c in emitted]), | |
| (torch.cat([c[1].state[0] for c in emitted], 1), | |
| torch.cat([c[1].state[1] for c in emitted], 1))) | |
| hyps, i = [], 0 | |
| for ys, (score, parent, token) in top: | |
| if token is None: | |
| parent.score = score | |
| hyps.append(parent) | |
| else: | |
| lm_state = parent.lm_step(lm, token)[1] if fuse else None | |
| hyps.append(Hyp(ys, score, y[i, 0], (hn[:, i : i + 1], cn[:, i : i + 1]), lm_state)) | |
| i += 1 | |
| final = [(h.score + (alpha * lm.end(h.lm_state) if fuse else 0.0), h.ys) for h in hyps] | |
| final.sort(key=lambda r: -r[0]) | |
| return final | |
| def joint_rescore(head, ctc: torch.nn.Module, enc: torch.Tensor, lam: float, beta: float, beam: int = 4) -> tuple[list[int], list]: | |
| """One clip's reader output (T, D) -> the J(lam, beta) pick (model ids) and the scored candidates | |
| [(source, tokens, log P_RNNT, log P_CTC)]. head: a Transducer; ctc: the CTC head's Linear. decode_clip's J branch.""" | |
| enc = enc.float() | |
| logp = ctc(enc).float().log_softmax(-1) | |
| c_toks, _ = ctc_greedy(logp.argmax(-1)) | |
| nbest = beam_decode(head, enc, beam=beam, style=None) | |
| cands, seen = [], set() | |
| for src, ys in [("beam", tuple(y)) for _, y in nbest] + [("ctc", tuple(c_toks))]: | |
| if ys not in seen: | |
| seen.add(ys) | |
| cands.append((src, list(ys))) | |
| proj = head.enc_proj(enc) | |
| scored = [(src, ys, rnnt_forced(head, proj, ys, None), ctc_forced(logp, ys)) for src, ys in cands] | |
| key = lambda c: (1 - lam) * c[2] + lam * c[3] + beta * len(c[1]) # noqa: E731 | |
| return max(scored, key=key)[1], scored | |