Automatic Speech Recognition
PyTorch
Safetensors
GGUF
MLX
Tamil
English
speech
asr
streaming
rnnt
ctc
tamil
tanglish
code-switching
indian-english
verisafe
Instructions to use junafinity/VeriSafe-Tanglish-1-STT with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use junafinity/VeriSafe-Tanglish-1-STT with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download junafinity/VeriSafe-Tanglish-1-STT --local-dir VeriSafe-Tanglish-1-STT
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
File size: 22,986 Bytes
ec01e2d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 | """MLX port of the stage 6 / stage 7 / VeriSafe-Tanglish-STT-1 / STT-2 model (one architecture): Gemma 4 audio tower + layer mix +
reader + CTC and RNN-T heads, one clip at a time. STT-1 has two head sets on the one encoder: Tamil (ctc_head, transducer) and
code-mix (ctc_head_cm, transducer_cm); the head methods take the state-dict prefix. STT-2 adds the code-mix reader fork: a second
post-tower stack (layer mix, in_norm, in_proj, position conv, reader, out_norm) under the prefix fork_cm., on the same tower;
encode(..., stack=FORK) reads it (model.py mix_cm/read_cm), encode(..., stack="") the Tamil stack (model.py mix/read).
A line-by-line port of src/indic_stt/model.py and transformers' Gemma4Audio* classes (transformers 5.22.1). Weights come from
stt2_model.mlx.safetensors under the PyTorch state-dict names; only conv and nothing else changes layout (see export_release.py).
The 8-bit file stt2_model.mlx8.safetensors holds the same tensors with the 198 2-D ones quantized (mx.quantize, affine, 8 bits,
group 64); load_weights dequantizes them once to float32, so IndicSTT runs the same float32 code on both files.
No padding mask: a clip is one array with no padded frames, which is how run_mlx.py feeds it.
Also ported: the start penalty and start cue of Transducer.greedy, and the J(l, b) CTC-assisted rescoring of the project's
scripts/decode_ctc_assist.py (rnnt_beam.beam_decode without LM, rnnt_forced, ctc_forced, ctc_greedy): the beam's joint and
log-softmax in float32, both forced scores in float64 (numpy), as the PyTorch code computes them.
"""
from __future__ import annotations
import heapq
import json
import math
import mlx.core as mx
import numpy as np
HID = 1024
HEADS = 8
HEAD_DIM = HID // HEADS
CHUNK = 12 # attention_chunk_size
PAST = 12 # attention_context_left - 1
CONTEXT = CHUNK + PAST # attention_context_right is 0
SOFTCAP = 50.0
INVALID = -1e9
GRAD_CLIP = 1e10
RMS_EPS = 1e-6
Q_SCALE = (HEAD_DIM ** -0.5) / math.log(2)
K_SCALE = math.log(1 + math.e) / math.log(2)
READER_DIM = 768
READER_HEADS = 12
READER_LAYERS = 6
TOWER_LAYERS = 12
STREAM_CHUNK = 12
MAX_SYMBOLS = 5
PRED = 640
JOINT_CELLS = 8192 # decode_ctc_assist.JOINT_CELLS: T x (U+1) cells per joint slice in rnnt_forced
FORK = "fork_cm." # STT-2: state-dict prefix of the code-mix reader fork (model.py FORK)
FORK_MODULES = ("layer_logits", "in_norm", "in_proj", "position", "reader", "out_norm") # the stack it copies (model.py)
QUANT = "quantization" # 8-bit file: safetensors metadata key, JSON {"mode": "affine", "bits": 8, "group_size": 64, "tensors": 198}
def fork_names(names) -> tuple[set, set]:
"""(fork_cm.* names in `names`, the fork_cm.* names a complete fork needs: one per post-tower stack tensor)."""
have = {k for k in names if k.startswith(FORK)}
want = {FORK + k for k in names if k.split(".")[0] in FORK_MODULES}
return have, want
def quant_stem(name: str) -> str:
"""Prefix of a quantized tensor's scales and biases: the name minus ".weight" (MLX QuantizedLinear's foo.weight, foo.scales,
foo.biases), or the whole name for the 2-D tensors not called .weight (in_proj_weight, the LSTM's weight_ih_l0, weight_hh_l0)."""
return name[: -len(".weight")] if name.endswith(".weight") else name
def load_weights(path) -> tuple[dict[str, mx.array], dict | None]:
"""mx.load a release file -> ({PyTorch name: array}, the file's quantization dict or None). A file without the QUANT metadata
(stt2_model.mlx.safetensors) comes back exactly as mx.load returns it. On the 8-bit file each quantized 2-D weight is stored
under its own name as mx.quantize's packed uint32, with <stem>.scales and <stem>.biases (float32) next to it; it is
dequantized here, once, to a float32 weight of the original shape. Refused: scales or packed tensors without the metadata,
the metadata without matching scales and biases, or a count that differs from the metadata's."""
weights, meta = mx.load(str(path), return_metadata=True)
name = getattr(path, "name", str(path))
scales = sorted(k for k in weights if k.endswith(".scales"))
if QUANT not in meta:
if scales or any(v.dtype == mx.uint32 for v in weights.values()):
raise SystemExit(f"{name}: quantized tensors without '{QUANT}' metadata: refused")
return weights, None
q = json.loads(meta[QUANT])
if q.get("mode") != "affine" or len(scales) != q.get("tensors"):
raise SystemExit(f"{name}: '{QUANT}' {q} does not match the file ({len(scales)} scales): refused")
out = {k: v for k, v in weights.items() if not k.endswith((".scales", ".biases"))}
for s in scales:
stem = s[: -len(".scales")]
w = stem + ".weight" if stem + ".weight" in weights else stem
if w not in weights or weights[w].dtype != mx.uint32 or stem + ".biases" not in weights:
raise SystemExit(f"{name}: {s} has no packed uint32 weight or no {stem}.biases: refused")
out[w] = mx.dequantize(weights[w], weights[s], weights[stem + ".biases"], group_size=q["group_size"], bits=q["bits"],
mode="affine")
left = [k for k in weights if k.endswith(".biases") and k[: -len(".biases")] + ".scales" not in weights]
if left or any(v.dtype == mx.uint32 for v in out.values()):
raise SystemExit(f"{name}: biases without scales or a packed tensor without scales ({left[:3]}): refused")
mx.eval(list(out.values()))
return out, q
def silu(x):
return x * mx.sigmoid(x)
def gelu(x):
return 0.5 * x * (1 + mx.erf(x / math.sqrt(2)))
def softplus(x):
return mx.logaddexp(x, mx.array(0.0, dtype=x.dtype))
def softmax(x, axis=-1):
return mx.softmax(x, axis=axis)
class IndicSTT:
def __init__(self, weights: dict[str, mx.array]):
self.w = weights
# rel_pos_enc: fixed sinusoids for 13 relative positions, 12 down to 0.
n = HID // 2
inc = math.log(10000.0) / (n - 1)
inv = mx.exp(mx.arange(n, dtype=mx.float32) * -inc)
pos = mx.arange(CONTEXT // 2, -1, -1, dtype=mx.float32)[:, None] * inv[None, :]
self.pos = mx.concatenate([mx.sin(pos), mx.cos(pos)], axis=-1)
# ---- small parts ----------------------------------------------------------------------
def clipped(self, x, name):
w = self.w
x = mx.clip(x, w[f"{name}.input_min"], w[f"{name}.input_max"])
x = x @ w[f"{name}.linear.weight"].T
return mx.clip(x, w[f"{name}.output_min"], w[f"{name}.output_max"])
def rms(self, x, name):
return x * mx.power(mx.mean(x * x, axis=-1, keepdims=True) + RMS_EPS, -0.5) * self.w[f"{name}.weight"]
def layer_norm(self, x, name, eps=1e-5, bias=True):
mean = mx.mean(x, axis=-1, keepdims=True)
var = mx.mean(mx.square(x - mean), axis=-1, keepdims=True)
y = (x - mean) * mx.power(var + eps, -0.5) * self.w[f"{name}.weight"]
return y + self.w[f"{name}.bias"] if bias else y
def linear(self, x, name, bias=True):
y = x @ self.w[f"{name}.weight"].T
return y + self.w[f"{name}.bias"] if bias else y
# ---- tower ----------------------------------------------------------------------------
def subsample(self, feats):
"""feats (1, F, 128) -> (1, T, 1024), T = ceil(ceil(F / 2) / 2). NHWC throughout: (B, time, mel, channels)."""
x = feats[..., None]
for i in range(2):
name = f"tower.subsample_conv_projection.layer{i}"
x = mx.conv2d(x, self.w[f"{name}.conv.weight"], stride=(2, 2), padding=(1, 1))
x = mx.maximum(self.layer_norm(x, f"{name}.norm", eps=RMS_EPS, bias=False), 0.0)
x = x.reshape(x.shape[0], x.shape[1], -1)
return self.linear(x, "tower.subsample_conv_projection.input_proj_linear", bias=False)
def feed_forward(self, x, name):
res = x
x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP)
x = self.rms(x, f"{name}.pre_layer_norm")
x = silu(self.clipped(x, f"{name}.ffw_layer_1"))
x = self.clipped(x, f"{name}.ffw_layer_2")
x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP)
x = self.rms(x, f"{name}.post_layer_norm")
return x * 0.5 + res
@staticmethod
def blocked_mask(T: int) -> mx.array:
"""True = blocked, shape (blocks, CHUNK, CONTEXT): a frame sees itself and the 11 frames before it, valid frames only.
The same as transformers' sliding window mask (left window attention_context_left - 1 = 12, dist < 12) in blocked form."""
blocks = (T + CHUNK - 1) // CHUNK
q = mx.arange(blocks * CHUNK).reshape(blocks, CHUNK, 1)
start = (mx.arange(blocks) * CHUNK)[:, None, None]
kv = start - PAST + mx.arange(CONTEXT)[None, None, :]
dist = q - kv
allowed = (dist >= 0) & (dist < PAST) & (kv >= 0) & (kv < T) & (q < T)
return ~allowed
def attention(self, x, name, mask):
B, T, _ = x.shape
shape = (B, T, HEADS, HEAD_DIM)
q = self.clipped(x, f"{name}.q_proj").reshape(shape)
k = self.clipped(x, f"{name}.k_proj").reshape(shape)
v = self.clipped(x, f"{name}.v_proj").reshape(shape)
q = q * Q_SCALE * softplus(self.w[f"{name}.per_dim_scale"])
k = k * K_SCALE
blocks = (T + CHUNK - 1) // CHUNK
pad = blocks * CHUNK - T
q = mx.pad(q, [(0, 0), (0, pad), (0, 0), (0, 0)]).reshape(B, blocks, CHUNK, HEADS, HEAD_DIM)
idx = (mx.arange(blocks) * CHUNK)[:, None] + mx.arange(CONTEXT)[None, :]
pad_ctx = [(0, 0), (PAST, CHUNK - 1), (0, 0), (0, 0)]
k = mx.pad(k, pad_ctx)[:, idx] # (B, blocks, CONTEXT, HEADS, HEAD_DIM)
v = mx.pad(v, pad_ctx)[:, idx]
rel = (self.pos @ self.w[f"{name}.relative_k_proj.weight"].T).reshape(-1, HEADS, HEAD_DIM) # (13, H, D)
queries = q.transpose(0, 3, 1, 2, 4) # (B, H, blocks, CHUNK, D)
ac = queries @ k.transpose(0, 3, 1, 4, 2) # (B, H, blocks, CHUNK, CONTEXT)
flat = queries.reshape(B, HEADS, -1, HEAD_DIM)
bd = (flat @ rel.transpose(1, 2, 0)).reshape(B, HEADS, blocks, CHUNK, rel.shape[0])
# _rel_shift: pad positions to CONTEXT + 1, flatten rows, cut, reshape; each row moves one place left.
bd = mx.pad(bd, [(0, 0), (0, 0), (0, 0), (0, 0), (0, CONTEXT + 1 - rel.shape[0])])
bd = bd.reshape(B, HEADS, blocks, CHUNK * (CONTEXT + 1))[..., : CHUNK * CONTEXT]
bd = bd.reshape(B, HEADS, blocks, CHUNK, CONTEXT)
weights = mx.tanh((ac + bd) / SOFTCAP) * SOFTCAP
weights = mx.where(mask, INVALID, weights)
weights = softmax(weights)
out = weights @ v.transpose(0, 3, 1, 2, 4) # (B, H, blocks, CHUNK, D)
out = out.transpose(0, 2, 3, 1, 4).reshape(B, blocks * CHUNK, -1)[:, :T]
return self.clipped(out, f"{name}.post")
def light_conv(self, x, name):
res = x
x = self.rms(x, f"{name}.pre_layer_norm")
x = self.clipped(x, f"{name}.linear_start")
a, b = mx.split(x, 2, axis=-1) # GLU
x = a * mx.sigmoid(b)
x = mx.pad(x, [(0, 0), (4, 0), (0, 0)]) # causal, kernel 5
x = mx.conv1d(x, self.w[f"{name}.depthwise_conv1d.weight"], groups=HID)
x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP)
x = self.rms(x, f"{name}.conv_norm")
x = silu(x)
return self.clipped(x, f"{name}.linear_end") + res
def layer(self, x, i, mask):
name = f"tower.layers.{i}"
x = self.feed_forward(x, f"{name}.feed_forward1")
res = x
x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP)
x = self.rms(x, f"{name}.norm_pre_attn")
x = self.attention(x, f"{name}.self_attn", mask)
x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP)
x = self.rms(x, f"{name}.norm_post_attn") + res
x = self.light_conv(x, f"{name}.lconv1d")
x = self.feed_forward(x, f"{name}.feed_forward2")
x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP)
return self.rms(x, f"{name}.norm_out")
# ---- reader ---------------------------------------------------------------------------
def reader_block(self, x, i, block, stack: str = ""):
name = f"{stack}reader.layers.{i}"
B, T, D = x.shape
y = self.layer_norm(x, f"{name}.norm1")
qkv = y @ self.w[f"{name}.self_attn.in_proj_weight"].T + self.w[f"{name}.self_attn.in_proj_bias"]
d = D // READER_HEADS
q, k, v = [t.reshape(B, T, READER_HEADS, d).transpose(0, 2, 1, 3) for t in mx.split(qkv, 3, axis=-1)]
a = (q @ k.transpose(0, 1, 3, 2)) / math.sqrt(d)
if block is not None:
a = mx.where(block, -mx.inf, a)
y = (softmax(a) @ v).transpose(0, 2, 1, 3).reshape(B, T, D)
x = x + self.linear(y, f"{name}.self_attn.out_proj")
y = self.layer_norm(x, f"{name}.norm2")
return x + self.linear(gelu(self.linear(y, f"{name}.linear1")), f"{name}.linear2")
def encode(self, feats, streaming: bool = False, stack: str = ""):
"""feats (1, F, 128) -> normalized reader output (1, T, 768). stack: the post-tower stack's state-dict prefix, "" (the
Tamil stack, both modes of a file without the fork) or FORK (STT-2's code-mix reader fork); the tower is the same."""
hs = self.subsample(feats)
T = hs.shape[1]
mask = self.blocked_mask(T)
weights = softmax(self.w[f"{stack}layer_logits"])
mixed = weights[0] * hs
for i in range(TOWER_LAYERS):
hs = self.layer(hs, i, mask)
mixed = mixed + weights[i + 1] * hs
mx.eval(hs, mixed) # keep the lazy graph small
x = self.linear(self.layer_norm(mixed, f"{stack}in_norm"), f"{stack}in_proj")
y = mx.pad(x, [(0, 0), (14, 0), (0, 0)])
y = mx.conv1d(y, self.w[f"{stack}position.conv.weight"], groups=READER_DIM) + self.w[f"{stack}position.conv.bias"]
x = x + gelu(y)
block = None
if streaming:
idx = mx.arange(T) // STREAM_CHUNK
block = idx[None, :] > idx[:, None] # True = may not attend
for i in range(READER_LAYERS):
x = self.reader_block(x, i, block, stack)
x = self.layer_norm(x, f"{stack}out_norm")
mx.eval(x)
return x
# ---- heads ----------------------------------------------------------------------------
def ctc_logits(self, enc, head: str = "ctc_head"):
return self.linear(enc, head)
def ctc_log_probs(self, enc, head: str = "ctc_head"):
"""(1, T, V) float32 log-softmax, as model.ctc_log_probs."""
logits = self.ctc_logits(enc, head)
return logits - mx.logsumexp(logits, axis=-1, keepdims=True)
def _lstm(self, x, h, c, head: str = "transducer"):
w = self.w
gates = (x @ w[f"{head}.lstm.weight_ih_l0"].T + w[f"{head}.lstm.bias_ih_l0"]
+ h @ w[f"{head}.lstm.weight_hh_l0"].T + w[f"{head}.lstm.bias_hh_l0"])
i, f, g, o = mx.split(gates, 4, axis=-1)
c = mx.sigmoid(f) * c + mx.sigmoid(i) * mx.tanh(g)
h = mx.sigmoid(o) * mx.tanh(c)
return h, c
def _predict(self, token: int, h, c, head: str = "transducer"):
h, c = self._lstm(self.w[f"{head}.embed.weight"][token], h, c, head)
return self.linear(h, f"{head}.pred_proj"), h, c
def rnnt_greedy(self, enc, head: str = "transducer", start_penalty: float = 0.0, cue=None) -> list[int]:
"""Transducer.greedy for one clip: each frame emits tokens until blank, at most MAX_SYMBOLS. start_penalty is taken off
blank's logit while nothing has been emitted, on the frames where cue (T,) bool is True (every frame if cue is None)."""
proj = self.linear(enc[0], f"{head}.enc_proj")
mx.eval(proj)
h = c = mx.zeros((PRED,))
pred, h, c = self._predict(0, h, c, head)
out = []
for t in range(proj.shape[0]):
e = proj[t]
for _ in range(MAX_SYMBOLS):
logits = self.linear(mx.tanh(e + pred), f"{head}.out")
if start_penalty and not out and (cue is None or cue[t]):
logits = mx.concatenate([logits[:1] - start_penalty, logits[1:]])
k = mx.argmax(logits)
mx.eval(k)
k = int(k.item())
if k == 0:
break
pred, h, c = self._predict(k, h, c, head)
mx.eval(pred, h, c)
out.append(k)
return out
# ---- J(l, b): CTC-assisted rescoring (scripts/decode_ctc_assist.py), one clip ---------------
def beam_decode(self, enc, head: str = "transducer", beam: int = 4, topk: int = 8, prune: float = 12.0):
"""rnnt_beam.beam_decode with no LM, bonus or mask, for one clip's reader output (T, D) -> all final hypotheses, best
first, as (score, tokens). A hypothesis is [tokens, score, pred, h, c]; merging, pruning and order as there."""
proj = self.linear(enc, f"{head}.enc_proj")
pred, h, c = self._predict(0, mx.zeros((PRED,)), mx.zeros((PRED,)), head)
mx.eval(proj, pred, h, c)
hyps = [[(), 0.0, pred, h, c]]
k_top = min(topk, self.w[f"{head}.out.weight"].shape[0] - 1)
for t in range(proj.shape[0]):
logits = self.linear(mx.tanh(proj[t][None] + mx.stack([x[2] for x in hyps])), f"{head}.out")
logp = np.array(logits - mx.logsumexp(logits, axis=-1, keepdims=True)) # (H, V) float32
tokens = np.argsort(-logp[:, 1:], axis=-1, kind="stable")[:, :k_top]
values = np.take_along_axis(logp[:, 1:], tokens, axis=-1)
blanks, values, tokens = logp[:, 0].tolist(), values.tolist(), tokens.tolist()
cands = {x[0]: [x[1] + b, x, None] for x, b in zip(hyps, blanks)}
for x, b, vs, ks in zip(hyps, blanks, values, tokens):
floor = max(b, vs[0]) - prune
for v, k in zip(vs, ks):
if v < floor or v == -math.inf:
break # sorted
k += 1
score = x[1] + v
ys = x[0] + (k,)
seen = cands.get(ys)
if seen is None:
cands[ys] = [score, x, k]
else: # an existing hypothesis that took blank here
seen[0] = _log_add(seen[0], score)
top = heapq.nlargest(beam, cands.items(), key=lambda kv: kv[1][0])
new = []
for ys, (score, parent, token) in top:
if token is None:
parent[1] = score
new.append(parent)
else:
p, hn, cn = self._predict(token, parent[3], parent[4], head)
mx.eval(p, hn, cn)
new.append([ys, score, p, hn, cn])
hyps = new
final = [(x[1], x[0]) for x in hyps]
final.sort(key=lambda r: -r[0])
return final
def rnnt_forced(self, proj, y: list[int], head: str = "transducer", cells: int = JOINT_CELLS) -> float:
"""decode_ctc_assist.rnnt_forced: log P_RNNT(y | x) over all alignments. proj = enc_proj(enc), (T, J). Joint float32,
log-softmax and the lattice recursion float64."""
T, U = proj.shape[0], len(y)
h = c = mx.zeros((PRED,))
preds = []
for k in [0] + list(y):
p, h, c = self._predict(k, h, c, head)
preds.append(p)
pred = mx.stack(preds) # (U+1, J)
blank = np.empty((T, U + 1), dtype=np.float64)
emit = np.empty((T, U), dtype=np.float64)
step = max(1, cells // (U + 1))
for t0 in range(0, T, step):
lg = np.array(self.linear(mx.tanh(proj[t0 : t0 + step, None] + pred[None]), f"{head}.out")).astype(np.float64)
m = lg.max(-1, keepdims=True)
lg = lg - (m + np.log(np.exp(lg - m).sum(-1, keepdims=True)))
blank[t0 : t0 + step] = lg[..., 0]
if U:
emit[t0 : t0 + step] = lg[:, np.arange(U), np.asarray(y)]
alpha = np.full(U + 1, -np.inf)
alpha[0] = 0.0
for t in range(T):
a = alpha if t == 0 else alpha + blank[t - 1] # from (t-1, u) by blank
cs = np.concatenate([[0.0], np.cumsum(emit[t])]) # tokens within frame t
alpha = cs + np.logaddexp.accumulate(a - cs)
return float(alpha[U] + blank[T - 1, U])
def joint_rescore(self, enc, head: str = "transducer", ctc: str = "ctc_head", lam: float = 0.3, beta: float = 0.0,
beam: int = 4) -> tuple[list[int], list]:
"""decode_ctc_assist.decode_clip's J(lam, beta) for one clip's reader output (T, D) -> (pick, scored candidates
[(source, tokens, log P_RNNT, log P_CTC)]). Ties keep the earlier candidate (beam order, then CTC)."""
logits = self.linear(enc, ctc)
logp = np.array(logits - mx.logsumexp(logits, axis=-1, keepdims=True)) # (T, V) float32
c_toks = ctc_greedy(logp.argmax(-1))
nbest = self.beam_decode(enc, head, beam)
cands, seen = [], set()
for src, ys in [("beam", tuple(y)) for _, y in nbest] + [("ctc", tuple(c_toks))]:
if ys not in seen:
seen.add(ys)
cands.append((src, list(ys)))
proj = self.linear(enc, f"{head}.enc_proj")
mx.eval(proj)
scored = [(src, ys, self.rnnt_forced(proj, ys, head), ctc_forced(logp, ys)) for src, ys in cands]
key = lambda s: (1 - lam) * s[2] + lam * s[3] + beta * len(s[1]) # noqa: E731
return max(scored, key=key)[1], scored
def _log_add(a: float, b: float) -> float:
return (a + math.log1p(math.exp(b - a))) if a >= b else (b + math.log1p(math.exp(a - b)))
def ctc_greedy(best) -> list[int]:
"""Frame argmax (T,) -> tokens: merge repeats, drop blanks (CTCTokenizer.greedy's rule)."""
toks, prev = [], -1
for k in np.asarray(best).tolist():
if k != prev and k != 0:
toks.append(k)
prev = k
return toks
def ctc_forced(logp: np.ndarray, y: list[int]) -> float:
"""decode_ctc_assist.ctc_forced: log P_CTC(y | x) over all alignments, the forward recursion in float64 on (T, V)
log-probs, blank 0. -inf if y cannot fit in T frames."""
T = logp.shape[0]
if not y:
return float(logp[:, 0].astype(np.float64).sum())
ext = np.zeros(2 * len(y) + 1, dtype=np.int64)
ext[1::2] = y
lp = logp[:, ext].astype(np.float64) # (T, S)
skip = np.zeros(len(ext), dtype=bool) # s-2 -> s: a token state whose token differs from the previous one
skip[3::2] = ext[3::2] != ext[1:-2:2]
alpha = np.full(len(ext), -np.inf)
alpha[:2] = lp[0, :2]
for t in range(1, T):
a1 = np.concatenate([[-np.inf], alpha[:-1]])
a2 = np.where(skip, np.concatenate([[-np.inf, -np.inf], alpha[:-2]]), -np.inf)
alpha = np.logaddexp(np.logaddexp(alpha, a1), a2) + lp[t]
return float(np.logaddexp(alpha[-1], alpha[-2]))
Stage6 = IndicSTT # the stage 6 release's name for the same class
|