"""MLX port of the stage 6 / stage 7 / VeriSafe-Tanglish-STT-1 / STT-2 model (one architecture): Gemma 4 audio tower + layer mix + reader + CTC and RNN-T heads, one clip at a time. STT-1 has two head sets on the one encoder: Tamil (ctc_head, transducer) and code-mix (ctc_head_cm, transducer_cm); the head methods take the state-dict prefix. STT-2 adds the code-mix reader fork: a second post-tower stack (layer mix, in_norm, in_proj, position conv, reader, out_norm) under the prefix fork_cm., on the same tower; encode(..., stack=FORK) reads it (model.py mix_cm/read_cm), encode(..., stack="") the Tamil stack (model.py mix/read). A line-by-line port of src/indic_stt/model.py and transformers' Gemma4Audio* classes (transformers 5.22.1). Weights come from stt2_model.mlx.safetensors under the PyTorch state-dict names; only conv and nothing else changes layout (see export_release.py). The 8-bit file stt2_model.mlx8.safetensors holds the same tensors with the 198 2-D ones quantized (mx.quantize, affine, 8 bits, group 64); load_weights dequantizes them once to float32, so IndicSTT runs the same float32 code on both files. No padding mask: a clip is one array with no padded frames, which is how run_mlx.py feeds it. Also ported: the start penalty and start cue of Transducer.greedy, and the J(l, b) CTC-assisted rescoring of the project's scripts/decode_ctc_assist.py (rnnt_beam.beam_decode without LM, rnnt_forced, ctc_forced, ctc_greedy): the beam's joint and log-softmax in float32, both forced scores in float64 (numpy), as the PyTorch code computes them. """ from __future__ import annotations import heapq import json import math import mlx.core as mx import numpy as np HID = 1024 HEADS = 8 HEAD_DIM = HID // HEADS CHUNK = 12 # attention_chunk_size PAST = 12 # attention_context_left - 1 CONTEXT = CHUNK + PAST # attention_context_right is 0 SOFTCAP = 50.0 INVALID = -1e9 GRAD_CLIP = 1e10 RMS_EPS = 1e-6 Q_SCALE = (HEAD_DIM ** -0.5) / math.log(2) K_SCALE = math.log(1 + math.e) / math.log(2) READER_DIM = 768 READER_HEADS = 12 READER_LAYERS = 6 TOWER_LAYERS = 12 STREAM_CHUNK = 12 MAX_SYMBOLS = 5 PRED = 640 JOINT_CELLS = 8192 # decode_ctc_assist.JOINT_CELLS: T x (U+1) cells per joint slice in rnnt_forced FORK = "fork_cm." # STT-2: state-dict prefix of the code-mix reader fork (model.py FORK) FORK_MODULES = ("layer_logits", "in_norm", "in_proj", "position", "reader", "out_norm") # the stack it copies (model.py) QUANT = "quantization" # 8-bit file: safetensors metadata key, JSON {"mode": "affine", "bits": 8, "group_size": 64, "tensors": 198} def fork_names(names) -> tuple[set, set]: """(fork_cm.* names in `names`, the fork_cm.* names a complete fork needs: one per post-tower stack tensor).""" have = {k for k in names if k.startswith(FORK)} want = {FORK + k for k in names if k.split(".")[0] in FORK_MODULES} return have, want def quant_stem(name: str) -> str: """Prefix of a quantized tensor's scales and biases: the name minus ".weight" (MLX QuantizedLinear's foo.weight, foo.scales, foo.biases), or the whole name for the 2-D tensors not called .weight (in_proj_weight, the LSTM's weight_ih_l0, weight_hh_l0).""" return name[: -len(".weight")] if name.endswith(".weight") else name def load_weights(path) -> tuple[dict[str, mx.array], dict | None]: """mx.load a release file -> ({PyTorch name: array}, the file's quantization dict or None). A file without the QUANT metadata (stt2_model.mlx.safetensors) comes back exactly as mx.load returns it. On the 8-bit file each quantized 2-D weight is stored under its own name as mx.quantize's packed uint32, with .scales and .biases (float32) next to it; it is dequantized here, once, to a float32 weight of the original shape. Refused: scales or packed tensors without the metadata, the metadata without matching scales and biases, or a count that differs from the metadata's.""" weights, meta = mx.load(str(path), return_metadata=True) name = getattr(path, "name", str(path)) scales = sorted(k for k in weights if k.endswith(".scales")) if QUANT not in meta: if scales or any(v.dtype == mx.uint32 for v in weights.values()): raise SystemExit(f"{name}: quantized tensors without '{QUANT}' metadata: refused") return weights, None q = json.loads(meta[QUANT]) if q.get("mode") != "affine" or len(scales) != q.get("tensors"): raise SystemExit(f"{name}: '{QUANT}' {q} does not match the file ({len(scales)} scales): refused") out = {k: v for k, v in weights.items() if not k.endswith((".scales", ".biases"))} for s in scales: stem = s[: -len(".scales")] w = stem + ".weight" if stem + ".weight" in weights else stem if w not in weights or weights[w].dtype != mx.uint32 or stem + ".biases" not in weights: raise SystemExit(f"{name}: {s} has no packed uint32 weight or no {stem}.biases: refused") out[w] = mx.dequantize(weights[w], weights[s], weights[stem + ".biases"], group_size=q["group_size"], bits=q["bits"], mode="affine") left = [k for k in weights if k.endswith(".biases") and k[: -len(".biases")] + ".scales" not in weights] if left or any(v.dtype == mx.uint32 for v in out.values()): raise SystemExit(f"{name}: biases without scales or a packed tensor without scales ({left[:3]}): refused") mx.eval(list(out.values())) return out, q def silu(x): return x * mx.sigmoid(x) def gelu(x): return 0.5 * x * (1 + mx.erf(x / math.sqrt(2))) def softplus(x): return mx.logaddexp(x, mx.array(0.0, dtype=x.dtype)) def softmax(x, axis=-1): return mx.softmax(x, axis=axis) class IndicSTT: def __init__(self, weights: dict[str, mx.array]): self.w = weights # rel_pos_enc: fixed sinusoids for 13 relative positions, 12 down to 0. n = HID // 2 inc = math.log(10000.0) / (n - 1) inv = mx.exp(mx.arange(n, dtype=mx.float32) * -inc) pos = mx.arange(CONTEXT // 2, -1, -1, dtype=mx.float32)[:, None] * inv[None, :] self.pos = mx.concatenate([mx.sin(pos), mx.cos(pos)], axis=-1) # ---- small parts ---------------------------------------------------------------------- def clipped(self, x, name): w = self.w x = mx.clip(x, w[f"{name}.input_min"], w[f"{name}.input_max"]) x = x @ w[f"{name}.linear.weight"].T return mx.clip(x, w[f"{name}.output_min"], w[f"{name}.output_max"]) def rms(self, x, name): return x * mx.power(mx.mean(x * x, axis=-1, keepdims=True) + RMS_EPS, -0.5) * self.w[f"{name}.weight"] def layer_norm(self, x, name, eps=1e-5, bias=True): mean = mx.mean(x, axis=-1, keepdims=True) var = mx.mean(mx.square(x - mean), axis=-1, keepdims=True) y = (x - mean) * mx.power(var + eps, -0.5) * self.w[f"{name}.weight"] return y + self.w[f"{name}.bias"] if bias else y def linear(self, x, name, bias=True): y = x @ self.w[f"{name}.weight"].T return y + self.w[f"{name}.bias"] if bias else y # ---- tower ---------------------------------------------------------------------------- def subsample(self, feats): """feats (1, F, 128) -> (1, T, 1024), T = ceil(ceil(F / 2) / 2). NHWC throughout: (B, time, mel, channels).""" x = feats[..., None] for i in range(2): name = f"tower.subsample_conv_projection.layer{i}" x = mx.conv2d(x, self.w[f"{name}.conv.weight"], stride=(2, 2), padding=(1, 1)) x = mx.maximum(self.layer_norm(x, f"{name}.norm", eps=RMS_EPS, bias=False), 0.0) x = x.reshape(x.shape[0], x.shape[1], -1) return self.linear(x, "tower.subsample_conv_projection.input_proj_linear", bias=False) def feed_forward(self, x, name): res = x x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP) x = self.rms(x, f"{name}.pre_layer_norm") x = silu(self.clipped(x, f"{name}.ffw_layer_1")) x = self.clipped(x, f"{name}.ffw_layer_2") x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP) x = self.rms(x, f"{name}.post_layer_norm") return x * 0.5 + res @staticmethod def blocked_mask(T: int) -> mx.array: """True = blocked, shape (blocks, CHUNK, CONTEXT): a frame sees itself and the 11 frames before it, valid frames only. The same as transformers' sliding window mask (left window attention_context_left - 1 = 12, dist < 12) in blocked form.""" blocks = (T + CHUNK - 1) // CHUNK q = mx.arange(blocks * CHUNK).reshape(blocks, CHUNK, 1) start = (mx.arange(blocks) * CHUNK)[:, None, None] kv = start - PAST + mx.arange(CONTEXT)[None, None, :] dist = q - kv allowed = (dist >= 0) & (dist < PAST) & (kv >= 0) & (kv < T) & (q < T) return ~allowed def attention(self, x, name, mask): B, T, _ = x.shape shape = (B, T, HEADS, HEAD_DIM) q = self.clipped(x, f"{name}.q_proj").reshape(shape) k = self.clipped(x, f"{name}.k_proj").reshape(shape) v = self.clipped(x, f"{name}.v_proj").reshape(shape) q = q * Q_SCALE * softplus(self.w[f"{name}.per_dim_scale"]) k = k * K_SCALE blocks = (T + CHUNK - 1) // CHUNK pad = blocks * CHUNK - T q = mx.pad(q, [(0, 0), (0, pad), (0, 0), (0, 0)]).reshape(B, blocks, CHUNK, HEADS, HEAD_DIM) idx = (mx.arange(blocks) * CHUNK)[:, None] + mx.arange(CONTEXT)[None, :] pad_ctx = [(0, 0), (PAST, CHUNK - 1), (0, 0), (0, 0)] k = mx.pad(k, pad_ctx)[:, idx] # (B, blocks, CONTEXT, HEADS, HEAD_DIM) v = mx.pad(v, pad_ctx)[:, idx] rel = (self.pos @ self.w[f"{name}.relative_k_proj.weight"].T).reshape(-1, HEADS, HEAD_DIM) # (13, H, D) queries = q.transpose(0, 3, 1, 2, 4) # (B, H, blocks, CHUNK, D) ac = queries @ k.transpose(0, 3, 1, 4, 2) # (B, H, blocks, CHUNK, CONTEXT) flat = queries.reshape(B, HEADS, -1, HEAD_DIM) bd = (flat @ rel.transpose(1, 2, 0)).reshape(B, HEADS, blocks, CHUNK, rel.shape[0]) # _rel_shift: pad positions to CONTEXT + 1, flatten rows, cut, reshape; each row moves one place left. bd = mx.pad(bd, [(0, 0), (0, 0), (0, 0), (0, 0), (0, CONTEXT + 1 - rel.shape[0])]) bd = bd.reshape(B, HEADS, blocks, CHUNK * (CONTEXT + 1))[..., : CHUNK * CONTEXT] bd = bd.reshape(B, HEADS, blocks, CHUNK, CONTEXT) weights = mx.tanh((ac + bd) / SOFTCAP) * SOFTCAP weights = mx.where(mask, INVALID, weights) weights = softmax(weights) out = weights @ v.transpose(0, 3, 1, 2, 4) # (B, H, blocks, CHUNK, D) out = out.transpose(0, 2, 3, 1, 4).reshape(B, blocks * CHUNK, -1)[:, :T] return self.clipped(out, f"{name}.post") def light_conv(self, x, name): res = x x = self.rms(x, f"{name}.pre_layer_norm") x = self.clipped(x, f"{name}.linear_start") a, b = mx.split(x, 2, axis=-1) # GLU x = a * mx.sigmoid(b) x = mx.pad(x, [(0, 0), (4, 0), (0, 0)]) # causal, kernel 5 x = mx.conv1d(x, self.w[f"{name}.depthwise_conv1d.weight"], groups=HID) x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP) x = self.rms(x, f"{name}.conv_norm") x = silu(x) return self.clipped(x, f"{name}.linear_end") + res def layer(self, x, i, mask): name = f"tower.layers.{i}" x = self.feed_forward(x, f"{name}.feed_forward1") res = x x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP) x = self.rms(x, f"{name}.norm_pre_attn") x = self.attention(x, f"{name}.self_attn", mask) x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP) x = self.rms(x, f"{name}.norm_post_attn") + res x = self.light_conv(x, f"{name}.lconv1d") x = self.feed_forward(x, f"{name}.feed_forward2") x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP) return self.rms(x, f"{name}.norm_out") # ---- reader --------------------------------------------------------------------------- def reader_block(self, x, i, block, stack: str = ""): name = f"{stack}reader.layers.{i}" B, T, D = x.shape y = self.layer_norm(x, f"{name}.norm1") qkv = y @ self.w[f"{name}.self_attn.in_proj_weight"].T + self.w[f"{name}.self_attn.in_proj_bias"] d = D // READER_HEADS q, k, v = [t.reshape(B, T, READER_HEADS, d).transpose(0, 2, 1, 3) for t in mx.split(qkv, 3, axis=-1)] a = (q @ k.transpose(0, 1, 3, 2)) / math.sqrt(d) if block is not None: a = mx.where(block, -mx.inf, a) y = (softmax(a) @ v).transpose(0, 2, 1, 3).reshape(B, T, D) x = x + self.linear(y, f"{name}.self_attn.out_proj") y = self.layer_norm(x, f"{name}.norm2") return x + self.linear(gelu(self.linear(y, f"{name}.linear1")), f"{name}.linear2") def encode(self, feats, streaming: bool = False, stack: str = ""): """feats (1, F, 128) -> normalized reader output (1, T, 768). stack: the post-tower stack's state-dict prefix, "" (the Tamil stack, both modes of a file without the fork) or FORK (STT-2's code-mix reader fork); the tower is the same.""" hs = self.subsample(feats) T = hs.shape[1] mask = self.blocked_mask(T) weights = softmax(self.w[f"{stack}layer_logits"]) mixed = weights[0] * hs for i in range(TOWER_LAYERS): hs = self.layer(hs, i, mask) mixed = mixed + weights[i + 1] * hs mx.eval(hs, mixed) # keep the lazy graph small x = self.linear(self.layer_norm(mixed, f"{stack}in_norm"), f"{stack}in_proj") y = mx.pad(x, [(0, 0), (14, 0), (0, 0)]) y = mx.conv1d(y, self.w[f"{stack}position.conv.weight"], groups=READER_DIM) + self.w[f"{stack}position.conv.bias"] x = x + gelu(y) block = None if streaming: idx = mx.arange(T) // STREAM_CHUNK block = idx[None, :] > idx[:, None] # True = may not attend for i in range(READER_LAYERS): x = self.reader_block(x, i, block, stack) x = self.layer_norm(x, f"{stack}out_norm") mx.eval(x) return x # ---- heads ---------------------------------------------------------------------------- def ctc_logits(self, enc, head: str = "ctc_head"): return self.linear(enc, head) def ctc_log_probs(self, enc, head: str = "ctc_head"): """(1, T, V) float32 log-softmax, as model.ctc_log_probs.""" logits = self.ctc_logits(enc, head) return logits - mx.logsumexp(logits, axis=-1, keepdims=True) def _lstm(self, x, h, c, head: str = "transducer"): w = self.w gates = (x @ w[f"{head}.lstm.weight_ih_l0"].T + w[f"{head}.lstm.bias_ih_l0"] + h @ w[f"{head}.lstm.weight_hh_l0"].T + w[f"{head}.lstm.bias_hh_l0"]) i, f, g, o = mx.split(gates, 4, axis=-1) c = mx.sigmoid(f) * c + mx.sigmoid(i) * mx.tanh(g) h = mx.sigmoid(o) * mx.tanh(c) return h, c def _predict(self, token: int, h, c, head: str = "transducer"): h, c = self._lstm(self.w[f"{head}.embed.weight"][token], h, c, head) return self.linear(h, f"{head}.pred_proj"), h, c def rnnt_greedy(self, enc, head: str = "transducer", start_penalty: float = 0.0, cue=None) -> list[int]: """Transducer.greedy for one clip: each frame emits tokens until blank, at most MAX_SYMBOLS. start_penalty is taken off blank's logit while nothing has been emitted, on the frames where cue (T,) bool is True (every frame if cue is None).""" proj = self.linear(enc[0], f"{head}.enc_proj") mx.eval(proj) h = c = mx.zeros((PRED,)) pred, h, c = self._predict(0, h, c, head) out = [] for t in range(proj.shape[0]): e = proj[t] for _ in range(MAX_SYMBOLS): logits = self.linear(mx.tanh(e + pred), f"{head}.out") if start_penalty and not out and (cue is None or cue[t]): logits = mx.concatenate([logits[:1] - start_penalty, logits[1:]]) k = mx.argmax(logits) mx.eval(k) k = int(k.item()) if k == 0: break pred, h, c = self._predict(k, h, c, head) mx.eval(pred, h, c) out.append(k) return out # ---- J(l, b): CTC-assisted rescoring (scripts/decode_ctc_assist.py), one clip --------------- def beam_decode(self, enc, head: str = "transducer", beam: int = 4, topk: int = 8, prune: float = 12.0): """rnnt_beam.beam_decode with no LM, bonus or mask, for one clip's reader output (T, D) -> all final hypotheses, best first, as (score, tokens). A hypothesis is [tokens, score, pred, h, c]; merging, pruning and order as there.""" proj = self.linear(enc, f"{head}.enc_proj") pred, h, c = self._predict(0, mx.zeros((PRED,)), mx.zeros((PRED,)), head) mx.eval(proj, pred, h, c) hyps = [[(), 0.0, pred, h, c]] k_top = min(topk, self.w[f"{head}.out.weight"].shape[0] - 1) for t in range(proj.shape[0]): logits = self.linear(mx.tanh(proj[t][None] + mx.stack([x[2] for x in hyps])), f"{head}.out") logp = np.array(logits - mx.logsumexp(logits, axis=-1, keepdims=True)) # (H, V) float32 tokens = np.argsort(-logp[:, 1:], axis=-1, kind="stable")[:, :k_top] values = np.take_along_axis(logp[:, 1:], tokens, axis=-1) blanks, values, tokens = logp[:, 0].tolist(), values.tolist(), tokens.tolist() cands = {x[0]: [x[1] + b, x, None] for x, b in zip(hyps, blanks)} for x, b, vs, ks in zip(hyps, blanks, values, tokens): floor = max(b, vs[0]) - prune for v, k in zip(vs, ks): if v < floor or v == -math.inf: break # sorted k += 1 score = x[1] + v ys = x[0] + (k,) seen = cands.get(ys) if seen is None: cands[ys] = [score, x, k] else: # an existing hypothesis that took blank here seen[0] = _log_add(seen[0], score) top = heapq.nlargest(beam, cands.items(), key=lambda kv: kv[1][0]) new = [] for ys, (score, parent, token) in top: if token is None: parent[1] = score new.append(parent) else: p, hn, cn = self._predict(token, parent[3], parent[4], head) mx.eval(p, hn, cn) new.append([ys, score, p, hn, cn]) hyps = new final = [(x[1], x[0]) for x in hyps] final.sort(key=lambda r: -r[0]) return final def rnnt_forced(self, proj, y: list[int], head: str = "transducer", cells: int = JOINT_CELLS) -> float: """decode_ctc_assist.rnnt_forced: log P_RNNT(y | x) over all alignments. proj = enc_proj(enc), (T, J). Joint float32, log-softmax and the lattice recursion float64.""" T, U = proj.shape[0], len(y) h = c = mx.zeros((PRED,)) preds = [] for k in [0] + list(y): p, h, c = self._predict(k, h, c, head) preds.append(p) pred = mx.stack(preds) # (U+1, J) blank = np.empty((T, U + 1), dtype=np.float64) emit = np.empty((T, U), dtype=np.float64) step = max(1, cells // (U + 1)) for t0 in range(0, T, step): lg = np.array(self.linear(mx.tanh(proj[t0 : t0 + step, None] + pred[None]), f"{head}.out")).astype(np.float64) m = lg.max(-1, keepdims=True) lg = lg - (m + np.log(np.exp(lg - m).sum(-1, keepdims=True))) blank[t0 : t0 + step] = lg[..., 0] if U: emit[t0 : t0 + step] = lg[:, np.arange(U), np.asarray(y)] alpha = np.full(U + 1, -np.inf) alpha[0] = 0.0 for t in range(T): a = alpha if t == 0 else alpha + blank[t - 1] # from (t-1, u) by blank cs = np.concatenate([[0.0], np.cumsum(emit[t])]) # tokens within frame t alpha = cs + np.logaddexp.accumulate(a - cs) return float(alpha[U] + blank[T - 1, U]) def joint_rescore(self, enc, head: str = "transducer", ctc: str = "ctc_head", lam: float = 0.3, beta: float = 0.0, beam: int = 4) -> tuple[list[int], list]: """decode_ctc_assist.decode_clip's J(lam, beta) for one clip's reader output (T, D) -> (pick, scored candidates [(source, tokens, log P_RNNT, log P_CTC)]). Ties keep the earlier candidate (beam order, then CTC).""" logits = self.linear(enc, ctc) logp = np.array(logits - mx.logsumexp(logits, axis=-1, keepdims=True)) # (T, V) float32 c_toks = ctc_greedy(logp.argmax(-1)) nbest = self.beam_decode(enc, head, beam) cands, seen = [], set() for src, ys in [("beam", tuple(y)) for _, y in nbest] + [("ctc", tuple(c_toks))]: if ys not in seen: seen.add(ys) cands.append((src, list(ys))) proj = self.linear(enc, f"{head}.enc_proj") mx.eval(proj) scored = [(src, ys, self.rnnt_forced(proj, ys, head), ctc_forced(logp, ys)) for src, ys in cands] key = lambda s: (1 - lam) * s[2] + lam * s[3] + beta * len(s[1]) # noqa: E731 return max(scored, key=key)[1], scored def _log_add(a: float, b: float) -> float: return (a + math.log1p(math.exp(b - a))) if a >= b else (b + math.log1p(math.exp(a - b))) def ctc_greedy(best) -> list[int]: """Frame argmax (T,) -> tokens: merge repeats, drop blanks (CTCTokenizer.greedy's rule).""" toks, prev = [], -1 for k in np.asarray(best).tolist(): if k != prev and k != 0: toks.append(k) prev = k return toks def ctc_forced(logp: np.ndarray, y: list[int]) -> float: """decode_ctc_assist.ctc_forced: log P_CTC(y | x) over all alignments, the forward recursion in float64 on (T, V) log-probs, blank 0. -inf if y cannot fit in T frames.""" T = logp.shape[0] if not y: return float(logp[:, 0].astype(np.float64).sum()) ext = np.zeros(2 * len(y) + 1, dtype=np.int64) ext[1::2] = y lp = logp[:, ext].astype(np.float64) # (T, S) skip = np.zeros(len(ext), dtype=bool) # s-2 -> s: a token state whose token differs from the previous one skip[3::2] = ext[3::2] != ext[1:-2:2] alpha = np.full(len(ext), -np.inf) alpha[:2] = lp[0, :2] for t in range(1, T): a1 = np.concatenate([[-np.inf], alpha[:-1]]) a2 = np.where(skip, np.concatenate([[-np.inf, -np.inf], alpha[:-2]]), -np.inf) alpha = np.logaddexp(np.logaddexp(alpha, a1), a2) + lp[t] return float(np.logaddexp(alpha[-1], alpha[-2])) Stage6 = IndicSTT # the stage 6 release's name for the same class