Automatic Speech Recognition
PyTorch
Safetensors
GGUF
MLX
Tamil
English
speech
asr
streaming
rnnt
ctc
tamil
tanglish
code-switching
indian-english
verisafe
Instructions to use junafinity/VeriSafe-Tanglish-1-STT with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use junafinity/VeriSafe-Tanglish-1-STT with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download junafinity/VeriSafe-Tanglish-1-STT --local-dir VeriSafe-Tanglish-1-STT
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download code/export/mlx_model.py from junafinity/VeriSafe-Tanglish-1-STT: direct link, hf CLI and curl.
- Browser
- Download file 23 kB
-
https://huggingface.co/junafinity/VeriSafe-Tanglish-1-STT/resolve/main/code/export/mlx_model.py
- Command line
-
hf download hf://junafinity/VeriSafe-Tanglish-1-STT/code/export/mlx_model.py
-
curl -L -o mlx_model.py https://huggingface.co/junafinity/VeriSafe-Tanglish-1-STT/resolve/main/code/export/mlx_model.py
23 kB
| """MLX port of the stage 6 / stage 7 / VeriSafe-Tanglish-STT-1 / STT-2 model (one architecture): Gemma 4 audio tower + layer mix + | |
| reader + CTC and RNN-T heads, one clip at a time. STT-1 has two head sets on the one encoder: Tamil (ctc_head, transducer) and | |
| code-mix (ctc_head_cm, transducer_cm); the head methods take the state-dict prefix. STT-2 adds the code-mix reader fork: a second | |
| post-tower stack (layer mix, in_norm, in_proj, position conv, reader, out_norm) under the prefix fork_cm., on the same tower; | |
| encode(..., stack=FORK) reads it (model.py mix_cm/read_cm), encode(..., stack="") the Tamil stack (model.py mix/read). | |
| A line-by-line port of src/indic_stt/model.py and transformers' Gemma4Audio* classes (transformers 5.22.1). Weights come from | |
| stt2_model.mlx.safetensors under the PyTorch state-dict names; only conv and nothing else changes layout (see export_release.py). | |
| The 8-bit file stt2_model.mlx8.safetensors holds the same tensors with the 198 2-D ones quantized (mx.quantize, affine, 8 bits, | |
| group 64); load_weights dequantizes them once to float32, so IndicSTT runs the same float32 code on both files. | |
| No padding mask: a clip is one array with no padded frames, which is how run_mlx.py feeds it. | |
| Also ported: the start penalty and start cue of Transducer.greedy, and the J(l, b) CTC-assisted rescoring of the project's | |
| scripts/decode_ctc_assist.py (rnnt_beam.beam_decode without LM, rnnt_forced, ctc_forced, ctc_greedy): the beam's joint and | |
| log-softmax in float32, both forced scores in float64 (numpy), as the PyTorch code computes them. | |
| """ | |
| from __future__ import annotations | |
| import heapq | |
| import json | |
| import math | |
| import mlx.core as mx | |
| import numpy as np | |
| HID = 1024 | |
| HEADS = 8 | |
| HEAD_DIM = HID // HEADS | |
| CHUNK = 12 # attention_chunk_size | |
| PAST = 12 # attention_context_left - 1 | |
| CONTEXT = CHUNK + PAST # attention_context_right is 0 | |
| SOFTCAP = 50.0 | |
| INVALID = -1e9 | |
| GRAD_CLIP = 1e10 | |
| RMS_EPS = 1e-6 | |
| Q_SCALE = (HEAD_DIM ** -0.5) / math.log(2) | |
| K_SCALE = math.log(1 + math.e) / math.log(2) | |
| READER_DIM = 768 | |
| READER_HEADS = 12 | |
| READER_LAYERS = 6 | |
| TOWER_LAYERS = 12 | |
| STREAM_CHUNK = 12 | |
| MAX_SYMBOLS = 5 | |
| PRED = 640 | |
| JOINT_CELLS = 8192 # decode_ctc_assist.JOINT_CELLS: T x (U+1) cells per joint slice in rnnt_forced | |
| FORK = "fork_cm." # STT-2: state-dict prefix of the code-mix reader fork (model.py FORK) | |
| FORK_MODULES = ("layer_logits", "in_norm", "in_proj", "position", "reader", "out_norm") # the stack it copies (model.py) | |
| QUANT = "quantization" # 8-bit file: safetensors metadata key, JSON {"mode": "affine", "bits": 8, "group_size": 64, "tensors": 198} | |
| def fork_names(names) -> tuple[set, set]: | |
| """(fork_cm.* names in `names`, the fork_cm.* names a complete fork needs: one per post-tower stack tensor).""" | |
| have = {k for k in names if k.startswith(FORK)} | |
| want = {FORK + k for k in names if k.split(".")[0] in FORK_MODULES} | |
| return have, want | |
| def quant_stem(name: str) -> str: | |
| """Prefix of a quantized tensor's scales and biases: the name minus ".weight" (MLX QuantizedLinear's foo.weight, foo.scales, | |
| foo.biases), or the whole name for the 2-D tensors not called .weight (in_proj_weight, the LSTM's weight_ih_l0, weight_hh_l0).""" | |
| return name[: -len(".weight")] if name.endswith(".weight") else name | |
| def load_weights(path) -> tuple[dict[str, mx.array], dict | None]: | |
| """mx.load a release file -> ({PyTorch name: array}, the file's quantization dict or None). A file without the QUANT metadata | |
| (stt2_model.mlx.safetensors) comes back exactly as mx.load returns it. On the 8-bit file each quantized 2-D weight is stored | |
| under its own name as mx.quantize's packed uint32, with <stem>.scales and <stem>.biases (float32) next to it; it is | |
| dequantized here, once, to a float32 weight of the original shape. Refused: scales or packed tensors without the metadata, | |
| the metadata without matching scales and biases, or a count that differs from the metadata's.""" | |
| weights, meta = mx.load(str(path), return_metadata=True) | |
| name = getattr(path, "name", str(path)) | |
| scales = sorted(k for k in weights if k.endswith(".scales")) | |
| if QUANT not in meta: | |
| if scales or any(v.dtype == mx.uint32 for v in weights.values()): | |
| raise SystemExit(f"{name}: quantized tensors without '{QUANT}' metadata: refused") | |
| return weights, None | |
| q = json.loads(meta[QUANT]) | |
| if q.get("mode") != "affine" or len(scales) != q.get("tensors"): | |
| raise SystemExit(f"{name}: '{QUANT}' {q} does not match the file ({len(scales)} scales): refused") | |
| out = {k: v for k, v in weights.items() if not k.endswith((".scales", ".biases"))} | |
| for s in scales: | |
| stem = s[: -len(".scales")] | |
| w = stem + ".weight" if stem + ".weight" in weights else stem | |
| if w not in weights or weights[w].dtype != mx.uint32 or stem + ".biases" not in weights: | |
| raise SystemExit(f"{name}: {s} has no packed uint32 weight or no {stem}.biases: refused") | |
| out[w] = mx.dequantize(weights[w], weights[s], weights[stem + ".biases"], group_size=q["group_size"], bits=q["bits"], | |
| mode="affine") | |
| left = [k for k in weights if k.endswith(".biases") and k[: -len(".biases")] + ".scales" not in weights] | |
| if left or any(v.dtype == mx.uint32 for v in out.values()): | |
| raise SystemExit(f"{name}: biases without scales or a packed tensor without scales ({left[:3]}): refused") | |
| mx.eval(list(out.values())) | |
| return out, q | |
| def silu(x): | |
| return x * mx.sigmoid(x) | |
| def gelu(x): | |
| return 0.5 * x * (1 + mx.erf(x / math.sqrt(2))) | |
| def softplus(x): | |
| return mx.logaddexp(x, mx.array(0.0, dtype=x.dtype)) | |
| def softmax(x, axis=-1): | |
| return mx.softmax(x, axis=axis) | |
| class IndicSTT: | |
| def __init__(self, weights: dict[str, mx.array]): | |
| self.w = weights | |
| # rel_pos_enc: fixed sinusoids for 13 relative positions, 12 down to 0. | |
| n = HID // 2 | |
| inc = math.log(10000.0) / (n - 1) | |
| inv = mx.exp(mx.arange(n, dtype=mx.float32) * -inc) | |
| pos = mx.arange(CONTEXT // 2, -1, -1, dtype=mx.float32)[:, None] * inv[None, :] | |
| self.pos = mx.concatenate([mx.sin(pos), mx.cos(pos)], axis=-1) | |
| # ---- small parts ---------------------------------------------------------------------- | |
| def clipped(self, x, name): | |
| w = self.w | |
| x = mx.clip(x, w[f"{name}.input_min"], w[f"{name}.input_max"]) | |
| x = x @ w[f"{name}.linear.weight"].T | |
| return mx.clip(x, w[f"{name}.output_min"], w[f"{name}.output_max"]) | |
| def rms(self, x, name): | |
| return x * mx.power(mx.mean(x * x, axis=-1, keepdims=True) + RMS_EPS, -0.5) * self.w[f"{name}.weight"] | |
| def layer_norm(self, x, name, eps=1e-5, bias=True): | |
| mean = mx.mean(x, axis=-1, keepdims=True) | |
| var = mx.mean(mx.square(x - mean), axis=-1, keepdims=True) | |
| y = (x - mean) * mx.power(var + eps, -0.5) * self.w[f"{name}.weight"] | |
| return y + self.w[f"{name}.bias"] if bias else y | |
| def linear(self, x, name, bias=True): | |
| y = x @ self.w[f"{name}.weight"].T | |
| return y + self.w[f"{name}.bias"] if bias else y | |
| # ---- tower ---------------------------------------------------------------------------- | |
| def subsample(self, feats): | |
| """feats (1, F, 128) -> (1, T, 1024), T = ceil(ceil(F / 2) / 2). NHWC throughout: (B, time, mel, channels).""" | |
| x = feats[..., None] | |
| for i in range(2): | |
| name = f"tower.subsample_conv_projection.layer{i}" | |
| x = mx.conv2d(x, self.w[f"{name}.conv.weight"], stride=(2, 2), padding=(1, 1)) | |
| x = mx.maximum(self.layer_norm(x, f"{name}.norm", eps=RMS_EPS, bias=False), 0.0) | |
| x = x.reshape(x.shape[0], x.shape[1], -1) | |
| return self.linear(x, "tower.subsample_conv_projection.input_proj_linear", bias=False) | |
| def feed_forward(self, x, name): | |
| res = x | |
| x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP) | |
| x = self.rms(x, f"{name}.pre_layer_norm") | |
| x = silu(self.clipped(x, f"{name}.ffw_layer_1")) | |
| x = self.clipped(x, f"{name}.ffw_layer_2") | |
| x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP) | |
| x = self.rms(x, f"{name}.post_layer_norm") | |
| return x * 0.5 + res | |
| def blocked_mask(T: int) -> mx.array: | |
| """True = blocked, shape (blocks, CHUNK, CONTEXT): a frame sees itself and the 11 frames before it, valid frames only. | |
| The same as transformers' sliding window mask (left window attention_context_left - 1 = 12, dist < 12) in blocked form.""" | |
| blocks = (T + CHUNK - 1) // CHUNK | |
| q = mx.arange(blocks * CHUNK).reshape(blocks, CHUNK, 1) | |
| start = (mx.arange(blocks) * CHUNK)[:, None, None] | |
| kv = start - PAST + mx.arange(CONTEXT)[None, None, :] | |
| dist = q - kv | |
| allowed = (dist >= 0) & (dist < PAST) & (kv >= 0) & (kv < T) & (q < T) | |
| return ~allowed | |
| def attention(self, x, name, mask): | |
| B, T, _ = x.shape | |
| shape = (B, T, HEADS, HEAD_DIM) | |
| q = self.clipped(x, f"{name}.q_proj").reshape(shape) | |
| k = self.clipped(x, f"{name}.k_proj").reshape(shape) | |
| v = self.clipped(x, f"{name}.v_proj").reshape(shape) | |
| q = q * Q_SCALE * softplus(self.w[f"{name}.per_dim_scale"]) | |
| k = k * K_SCALE | |
| blocks = (T + CHUNK - 1) // CHUNK | |
| pad = blocks * CHUNK - T | |
| q = mx.pad(q, [(0, 0), (0, pad), (0, 0), (0, 0)]).reshape(B, blocks, CHUNK, HEADS, HEAD_DIM) | |
| idx = (mx.arange(blocks) * CHUNK)[:, None] + mx.arange(CONTEXT)[None, :] | |
| pad_ctx = [(0, 0), (PAST, CHUNK - 1), (0, 0), (0, 0)] | |
| k = mx.pad(k, pad_ctx)[:, idx] # (B, blocks, CONTEXT, HEADS, HEAD_DIM) | |
| v = mx.pad(v, pad_ctx)[:, idx] | |
| rel = (self.pos @ self.w[f"{name}.relative_k_proj.weight"].T).reshape(-1, HEADS, HEAD_DIM) # (13, H, D) | |
| queries = q.transpose(0, 3, 1, 2, 4) # (B, H, blocks, CHUNK, D) | |
| ac = queries @ k.transpose(0, 3, 1, 4, 2) # (B, H, blocks, CHUNK, CONTEXT) | |
| flat = queries.reshape(B, HEADS, -1, HEAD_DIM) | |
| bd = (flat @ rel.transpose(1, 2, 0)).reshape(B, HEADS, blocks, CHUNK, rel.shape[0]) | |
| # _rel_shift: pad positions to CONTEXT + 1, flatten rows, cut, reshape; each row moves one place left. | |
| bd = mx.pad(bd, [(0, 0), (0, 0), (0, 0), (0, 0), (0, CONTEXT + 1 - rel.shape[0])]) | |
| bd = bd.reshape(B, HEADS, blocks, CHUNK * (CONTEXT + 1))[..., : CHUNK * CONTEXT] | |
| bd = bd.reshape(B, HEADS, blocks, CHUNK, CONTEXT) | |
| weights = mx.tanh((ac + bd) / SOFTCAP) * SOFTCAP | |
| weights = mx.where(mask, INVALID, weights) | |
| weights = softmax(weights) | |
| out = weights @ v.transpose(0, 3, 1, 2, 4) # (B, H, blocks, CHUNK, D) | |
| out = out.transpose(0, 2, 3, 1, 4).reshape(B, blocks * CHUNK, -1)[:, :T] | |
| return self.clipped(out, f"{name}.post") | |
| def light_conv(self, x, name): | |
| res = x | |
| x = self.rms(x, f"{name}.pre_layer_norm") | |
| x = self.clipped(x, f"{name}.linear_start") | |
| a, b = mx.split(x, 2, axis=-1) # GLU | |
| x = a * mx.sigmoid(b) | |
| x = mx.pad(x, [(0, 0), (4, 0), (0, 0)]) # causal, kernel 5 | |
| x = mx.conv1d(x, self.w[f"{name}.depthwise_conv1d.weight"], groups=HID) | |
| x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP) | |
| x = self.rms(x, f"{name}.conv_norm") | |
| x = silu(x) | |
| return self.clipped(x, f"{name}.linear_end") + res | |
| def layer(self, x, i, mask): | |
| name = f"tower.layers.{i}" | |
| x = self.feed_forward(x, f"{name}.feed_forward1") | |
| res = x | |
| x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP) | |
| x = self.rms(x, f"{name}.norm_pre_attn") | |
| x = self.attention(x, f"{name}.self_attn", mask) | |
| x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP) | |
| x = self.rms(x, f"{name}.norm_post_attn") + res | |
| x = self.light_conv(x, f"{name}.lconv1d") | |
| x = self.feed_forward(x, f"{name}.feed_forward2") | |
| x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP) | |
| return self.rms(x, f"{name}.norm_out") | |
| # ---- reader --------------------------------------------------------------------------- | |
| def reader_block(self, x, i, block, stack: str = ""): | |
| name = f"{stack}reader.layers.{i}" | |
| B, T, D = x.shape | |
| y = self.layer_norm(x, f"{name}.norm1") | |
| qkv = y @ self.w[f"{name}.self_attn.in_proj_weight"].T + self.w[f"{name}.self_attn.in_proj_bias"] | |
| d = D // READER_HEADS | |
| q, k, v = [t.reshape(B, T, READER_HEADS, d).transpose(0, 2, 1, 3) for t in mx.split(qkv, 3, axis=-1)] | |
| a = (q @ k.transpose(0, 1, 3, 2)) / math.sqrt(d) | |
| if block is not None: | |
| a = mx.where(block, -mx.inf, a) | |
| y = (softmax(a) @ v).transpose(0, 2, 1, 3).reshape(B, T, D) | |
| x = x + self.linear(y, f"{name}.self_attn.out_proj") | |
| y = self.layer_norm(x, f"{name}.norm2") | |
| return x + self.linear(gelu(self.linear(y, f"{name}.linear1")), f"{name}.linear2") | |
| def encode(self, feats, streaming: bool = False, stack: str = ""): | |
| """feats (1, F, 128) -> normalized reader output (1, T, 768). stack: the post-tower stack's state-dict prefix, "" (the | |
| Tamil stack, both modes of a file without the fork) or FORK (STT-2's code-mix reader fork); the tower is the same.""" | |
| hs = self.subsample(feats) | |
| T = hs.shape[1] | |
| mask = self.blocked_mask(T) | |
| weights = softmax(self.w[f"{stack}layer_logits"]) | |
| mixed = weights[0] * hs | |
| for i in range(TOWER_LAYERS): | |
| hs = self.layer(hs, i, mask) | |
| mixed = mixed + weights[i + 1] * hs | |
| mx.eval(hs, mixed) # keep the lazy graph small | |
| x = self.linear(self.layer_norm(mixed, f"{stack}in_norm"), f"{stack}in_proj") | |
| y = mx.pad(x, [(0, 0), (14, 0), (0, 0)]) | |
| y = mx.conv1d(y, self.w[f"{stack}position.conv.weight"], groups=READER_DIM) + self.w[f"{stack}position.conv.bias"] | |
| x = x + gelu(y) | |
| block = None | |
| if streaming: | |
| idx = mx.arange(T) // STREAM_CHUNK | |
| block = idx[None, :] > idx[:, None] # True = may not attend | |
| for i in range(READER_LAYERS): | |
| x = self.reader_block(x, i, block, stack) | |
| x = self.layer_norm(x, f"{stack}out_norm") | |
| mx.eval(x) | |
| return x | |
| # ---- heads ---------------------------------------------------------------------------- | |
| def ctc_logits(self, enc, head: str = "ctc_head"): | |
| return self.linear(enc, head) | |
| def ctc_log_probs(self, enc, head: str = "ctc_head"): | |
| """(1, T, V) float32 log-softmax, as model.ctc_log_probs.""" | |
| logits = self.ctc_logits(enc, head) | |
| return logits - mx.logsumexp(logits, axis=-1, keepdims=True) | |
| def _lstm(self, x, h, c, head: str = "transducer"): | |
| w = self.w | |
| gates = (x @ w[f"{head}.lstm.weight_ih_l0"].T + w[f"{head}.lstm.bias_ih_l0"] | |
| + h @ w[f"{head}.lstm.weight_hh_l0"].T + w[f"{head}.lstm.bias_hh_l0"]) | |
| i, f, g, o = mx.split(gates, 4, axis=-1) | |
| c = mx.sigmoid(f) * c + mx.sigmoid(i) * mx.tanh(g) | |
| h = mx.sigmoid(o) * mx.tanh(c) | |
| return h, c | |
| def _predict(self, token: int, h, c, head: str = "transducer"): | |
| h, c = self._lstm(self.w[f"{head}.embed.weight"][token], h, c, head) | |
| return self.linear(h, f"{head}.pred_proj"), h, c | |
| def rnnt_greedy(self, enc, head: str = "transducer", start_penalty: float = 0.0, cue=None) -> list[int]: | |
| """Transducer.greedy for one clip: each frame emits tokens until blank, at most MAX_SYMBOLS. start_penalty is taken off | |
| blank's logit while nothing has been emitted, on the frames where cue (T,) bool is True (every frame if cue is None).""" | |
| proj = self.linear(enc[0], f"{head}.enc_proj") | |
| mx.eval(proj) | |
| h = c = mx.zeros((PRED,)) | |
| pred, h, c = self._predict(0, h, c, head) | |
| out = [] | |
| for t in range(proj.shape[0]): | |
| e = proj[t] | |
| for _ in range(MAX_SYMBOLS): | |
| logits = self.linear(mx.tanh(e + pred), f"{head}.out") | |
| if start_penalty and not out and (cue is None or cue[t]): | |
| logits = mx.concatenate([logits[:1] - start_penalty, logits[1:]]) | |
| k = mx.argmax(logits) | |
| mx.eval(k) | |
| k = int(k.item()) | |
| if k == 0: | |
| break | |
| pred, h, c = self._predict(k, h, c, head) | |
| mx.eval(pred, h, c) | |
| out.append(k) | |
| return out | |
| # ---- J(l, b): CTC-assisted rescoring (scripts/decode_ctc_assist.py), one clip --------------- | |
| def beam_decode(self, enc, head: str = "transducer", beam: int = 4, topk: int = 8, prune: float = 12.0): | |
| """rnnt_beam.beam_decode with no LM, bonus or mask, for one clip's reader output (T, D) -> all final hypotheses, best | |
| first, as (score, tokens). A hypothesis is [tokens, score, pred, h, c]; merging, pruning and order as there.""" | |
| proj = self.linear(enc, f"{head}.enc_proj") | |
| pred, h, c = self._predict(0, mx.zeros((PRED,)), mx.zeros((PRED,)), head) | |
| mx.eval(proj, pred, h, c) | |
| hyps = [[(), 0.0, pred, h, c]] | |
| k_top = min(topk, self.w[f"{head}.out.weight"].shape[0] - 1) | |
| for t in range(proj.shape[0]): | |
| logits = self.linear(mx.tanh(proj[t][None] + mx.stack([x[2] for x in hyps])), f"{head}.out") | |
| logp = np.array(logits - mx.logsumexp(logits, axis=-1, keepdims=True)) # (H, V) float32 | |
| tokens = np.argsort(-logp[:, 1:], axis=-1, kind="stable")[:, :k_top] | |
| values = np.take_along_axis(logp[:, 1:], tokens, axis=-1) | |
| blanks, values, tokens = logp[:, 0].tolist(), values.tolist(), tokens.tolist() | |
| cands = {x[0]: [x[1] + b, x, None] for x, b in zip(hyps, blanks)} | |
| for x, b, vs, ks in zip(hyps, blanks, values, tokens): | |
| floor = max(b, vs[0]) - prune | |
| for v, k in zip(vs, ks): | |
| if v < floor or v == -math.inf: | |
| break # sorted | |
| k += 1 | |
| score = x[1] + v | |
| ys = x[0] + (k,) | |
| seen = cands.get(ys) | |
| if seen is None: | |
| cands[ys] = [score, x, k] | |
| else: # an existing hypothesis that took blank here | |
| seen[0] = _log_add(seen[0], score) | |
| top = heapq.nlargest(beam, cands.items(), key=lambda kv: kv[1][0]) | |
| new = [] | |
| for ys, (score, parent, token) in top: | |
| if token is None: | |
| parent[1] = score | |
| new.append(parent) | |
| else: | |
| p, hn, cn = self._predict(token, parent[3], parent[4], head) | |
| mx.eval(p, hn, cn) | |
| new.append([ys, score, p, hn, cn]) | |
| hyps = new | |
| final = [(x[1], x[0]) for x in hyps] | |
| final.sort(key=lambda r: -r[0]) | |
| return final | |
| def rnnt_forced(self, proj, y: list[int], head: str = "transducer", cells: int = JOINT_CELLS) -> float: | |
| """decode_ctc_assist.rnnt_forced: log P_RNNT(y | x) over all alignments. proj = enc_proj(enc), (T, J). Joint float32, | |
| log-softmax and the lattice recursion float64.""" | |
| T, U = proj.shape[0], len(y) | |
| h = c = mx.zeros((PRED,)) | |
| preds = [] | |
| for k in [0] + list(y): | |
| p, h, c = self._predict(k, h, c, head) | |
| preds.append(p) | |
| pred = mx.stack(preds) # (U+1, J) | |
| blank = np.empty((T, U + 1), dtype=np.float64) | |
| emit = np.empty((T, U), dtype=np.float64) | |
| step = max(1, cells // (U + 1)) | |
| for t0 in range(0, T, step): | |
| lg = np.array(self.linear(mx.tanh(proj[t0 : t0 + step, None] + pred[None]), f"{head}.out")).astype(np.float64) | |
| m = lg.max(-1, keepdims=True) | |
| lg = lg - (m + np.log(np.exp(lg - m).sum(-1, keepdims=True))) | |
| blank[t0 : t0 + step] = lg[..., 0] | |
| if U: | |
| emit[t0 : t0 + step] = lg[:, np.arange(U), np.asarray(y)] | |
| alpha = np.full(U + 1, -np.inf) | |
| alpha[0] = 0.0 | |
| for t in range(T): | |
| a = alpha if t == 0 else alpha + blank[t - 1] # from (t-1, u) by blank | |
| cs = np.concatenate([[0.0], np.cumsum(emit[t])]) # tokens within frame t | |
| alpha = cs + np.logaddexp.accumulate(a - cs) | |
| return float(alpha[U] + blank[T - 1, U]) | |
| def joint_rescore(self, enc, head: str = "transducer", ctc: str = "ctc_head", lam: float = 0.3, beta: float = 0.0, | |
| beam: int = 4) -> tuple[list[int], list]: | |
| """decode_ctc_assist.decode_clip's J(lam, beta) for one clip's reader output (T, D) -> (pick, scored candidates | |
| [(source, tokens, log P_RNNT, log P_CTC)]). Ties keep the earlier candidate (beam order, then CTC).""" | |
| logits = self.linear(enc, ctc) | |
| logp = np.array(logits - mx.logsumexp(logits, axis=-1, keepdims=True)) # (T, V) float32 | |
| c_toks = ctc_greedy(logp.argmax(-1)) | |
| nbest = self.beam_decode(enc, head, beam) | |
| cands, seen = [], set() | |
| for src, ys in [("beam", tuple(y)) for _, y in nbest] + [("ctc", tuple(c_toks))]: | |
| if ys not in seen: | |
| seen.add(ys) | |
| cands.append((src, list(ys))) | |
| proj = self.linear(enc, f"{head}.enc_proj") | |
| mx.eval(proj) | |
| scored = [(src, ys, self.rnnt_forced(proj, ys, head), ctc_forced(logp, ys)) for src, ys in cands] | |
| key = lambda s: (1 - lam) * s[2] + lam * s[3] + beta * len(s[1]) # noqa: E731 | |
| return max(scored, key=key)[1], scored | |
| def _log_add(a: float, b: float) -> float: | |
| return (a + math.log1p(math.exp(b - a))) if a >= b else (b + math.log1p(math.exp(a - b))) | |
| def ctc_greedy(best) -> list[int]: | |
| """Frame argmax (T,) -> tokens: merge repeats, drop blanks (CTCTokenizer.greedy's rule).""" | |
| toks, prev = [], -1 | |
| for k in np.asarray(best).tolist(): | |
| if k != prev and k != 0: | |
| toks.append(k) | |
| prev = k | |
| return toks | |
| def ctc_forced(logp: np.ndarray, y: list[int]) -> float: | |
| """decode_ctc_assist.ctc_forced: log P_CTC(y | x) over all alignments, the forward recursion in float64 on (T, V) | |
| log-probs, blank 0. -inf if y cannot fit in T frames.""" | |
| T = logp.shape[0] | |
| if not y: | |
| return float(logp[:, 0].astype(np.float64).sum()) | |
| ext = np.zeros(2 * len(y) + 1, dtype=np.int64) | |
| ext[1::2] = y | |
| lp = logp[:, ext].astype(np.float64) # (T, S) | |
| skip = np.zeros(len(ext), dtype=bool) # s-2 -> s: a token state whose token differs from the previous one | |
| skip[3::2] = ext[3::2] != ext[1:-2:2] | |
| alpha = np.full(len(ext), -np.inf) | |
| alpha[:2] = lp[0, :2] | |
| for t in range(1, T): | |
| a1 = np.concatenate([[-np.inf], alpha[:-1]]) | |
| a2 = np.where(skip, np.concatenate([[-np.inf, -np.inf], alpha[:-2]]), -np.inf) | |
| alpha = np.logaddexp(np.logaddexp(alpha, a1), a2) + lp[t] | |
| return float(np.logaddexp(alpha[-1], alpha[-2])) | |
| Stage6 = IndicSTT # the stage 6 release's name for the same class | |