File size: 22,986 Bytes
ec01e2d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
"""MLX port of the stage 6 / stage 7 / VeriSafe-Tanglish-STT-1 / STT-2 model (one architecture): Gemma 4 audio tower + layer mix +
reader + CTC and RNN-T heads, one clip at a time. STT-1 has two head sets on the one encoder: Tamil (ctc_head, transducer) and
code-mix (ctc_head_cm, transducer_cm); the head methods take the state-dict prefix. STT-2 adds the code-mix reader fork: a second
post-tower stack (layer mix, in_norm, in_proj, position conv, reader, out_norm) under the prefix fork_cm., on the same tower;
encode(..., stack=FORK) reads it (model.py mix_cm/read_cm), encode(..., stack="") the Tamil stack (model.py mix/read).

A line-by-line port of src/indic_stt/model.py and transformers' Gemma4Audio* classes (transformers 5.22.1). Weights come from
stt2_model.mlx.safetensors under the PyTorch state-dict names; only conv and nothing else changes layout (see export_release.py).
The 8-bit file stt2_model.mlx8.safetensors holds the same tensors with the 198 2-D ones quantized (mx.quantize, affine, 8 bits,
group 64); load_weights dequantizes them once to float32, so IndicSTT runs the same float32 code on both files.
No padding mask: a clip is one array with no padded frames, which is how run_mlx.py feeds it.

Also ported: the start penalty and start cue of Transducer.greedy, and the J(l, b) CTC-assisted rescoring of the project's
scripts/decode_ctc_assist.py (rnnt_beam.beam_decode without LM, rnnt_forced, ctc_forced, ctc_greedy): the beam's joint and
log-softmax in float32, both forced scores in float64 (numpy), as the PyTorch code computes them.
"""
from __future__ import annotations

import heapq
import json
import math

import mlx.core as mx
import numpy as np

HID = 1024
HEADS = 8
HEAD_DIM = HID // HEADS
CHUNK = 12  # attention_chunk_size
PAST = 12  # attention_context_left - 1
CONTEXT = CHUNK + PAST  # attention_context_right is 0
SOFTCAP = 50.0
INVALID = -1e9
GRAD_CLIP = 1e10
RMS_EPS = 1e-6
Q_SCALE = (HEAD_DIM ** -0.5) / math.log(2)
K_SCALE = math.log(1 + math.e) / math.log(2)
READER_DIM = 768
READER_HEADS = 12
READER_LAYERS = 6
TOWER_LAYERS = 12
STREAM_CHUNK = 12
MAX_SYMBOLS = 5
PRED = 640
JOINT_CELLS = 8192  # decode_ctc_assist.JOINT_CELLS: T x (U+1) cells per joint slice in rnnt_forced
FORK = "fork_cm."  # STT-2: state-dict prefix of the code-mix reader fork (model.py FORK)
FORK_MODULES = ("layer_logits", "in_norm", "in_proj", "position", "reader", "out_norm")  # the stack it copies (model.py)
QUANT = "quantization"  # 8-bit file: safetensors metadata key, JSON {"mode": "affine", "bits": 8, "group_size": 64, "tensors": 198}


def fork_names(names) -> tuple[set, set]:
    """(fork_cm.* names in `names`, the fork_cm.* names a complete fork needs: one per post-tower stack tensor)."""
    have = {k for k in names if k.startswith(FORK)}
    want = {FORK + k for k in names if k.split(".")[0] in FORK_MODULES}
    return have, want


def quant_stem(name: str) -> str:
    """Prefix of a quantized tensor's scales and biases: the name minus ".weight" (MLX QuantizedLinear's foo.weight, foo.scales,
    foo.biases), or the whole name for the 2-D tensors not called .weight (in_proj_weight, the LSTM's weight_ih_l0, weight_hh_l0)."""
    return name[: -len(".weight")] if name.endswith(".weight") else name


def load_weights(path) -> tuple[dict[str, mx.array], dict | None]:
    """mx.load a release file -> ({PyTorch name: array}, the file's quantization dict or None). A file without the QUANT metadata
    (stt2_model.mlx.safetensors) comes back exactly as mx.load returns it. On the 8-bit file each quantized 2-D weight is stored
    under its own name as mx.quantize's packed uint32, with <stem>.scales and <stem>.biases (float32) next to it; it is
    dequantized here, once, to a float32 weight of the original shape. Refused: scales or packed tensors without the metadata,
    the metadata without matching scales and biases, or a count that differs from the metadata's."""
    weights, meta = mx.load(str(path), return_metadata=True)
    name = getattr(path, "name", str(path))
    scales = sorted(k for k in weights if k.endswith(".scales"))
    if QUANT not in meta:
        if scales or any(v.dtype == mx.uint32 for v in weights.values()):
            raise SystemExit(f"{name}: quantized tensors without '{QUANT}' metadata: refused")
        return weights, None
    q = json.loads(meta[QUANT])
    if q.get("mode") != "affine" or len(scales) != q.get("tensors"):
        raise SystemExit(f"{name}: '{QUANT}' {q} does not match the file ({len(scales)} scales): refused")
    out = {k: v for k, v in weights.items() if not k.endswith((".scales", ".biases"))}
    for s in scales:
        stem = s[: -len(".scales")]
        w = stem + ".weight" if stem + ".weight" in weights else stem
        if w not in weights or weights[w].dtype != mx.uint32 or stem + ".biases" not in weights:
            raise SystemExit(f"{name}: {s} has no packed uint32 weight or no {stem}.biases: refused")
        out[w] = mx.dequantize(weights[w], weights[s], weights[stem + ".biases"], group_size=q["group_size"], bits=q["bits"],
                               mode="affine")
    left = [k for k in weights if k.endswith(".biases") and k[: -len(".biases")] + ".scales" not in weights]
    if left or any(v.dtype == mx.uint32 for v in out.values()):
        raise SystemExit(f"{name}: biases without scales or a packed tensor without scales ({left[:3]}): refused")
    mx.eval(list(out.values()))
    return out, q


def silu(x):
    return x * mx.sigmoid(x)


def gelu(x):
    return 0.5 * x * (1 + mx.erf(x / math.sqrt(2)))


def softplus(x):
    return mx.logaddexp(x, mx.array(0.0, dtype=x.dtype))


def softmax(x, axis=-1):
    return mx.softmax(x, axis=axis)


class IndicSTT:
    def __init__(self, weights: dict[str, mx.array]):
        self.w = weights
        # rel_pos_enc: fixed sinusoids for 13 relative positions, 12 down to 0.
        n = HID // 2
        inc = math.log(10000.0) / (n - 1)
        inv = mx.exp(mx.arange(n, dtype=mx.float32) * -inc)
        pos = mx.arange(CONTEXT // 2, -1, -1, dtype=mx.float32)[:, None] * inv[None, :]
        self.pos = mx.concatenate([mx.sin(pos), mx.cos(pos)], axis=-1)

    # ---- small parts ----------------------------------------------------------------------
    def clipped(self, x, name):
        w = self.w
        x = mx.clip(x, w[f"{name}.input_min"], w[f"{name}.input_max"])
        x = x @ w[f"{name}.linear.weight"].T
        return mx.clip(x, w[f"{name}.output_min"], w[f"{name}.output_max"])

    def rms(self, x, name):
        return x * mx.power(mx.mean(x * x, axis=-1, keepdims=True) + RMS_EPS, -0.5) * self.w[f"{name}.weight"]

    def layer_norm(self, x, name, eps=1e-5, bias=True):
        mean = mx.mean(x, axis=-1, keepdims=True)
        var = mx.mean(mx.square(x - mean), axis=-1, keepdims=True)
        y = (x - mean) * mx.power(var + eps, -0.5) * self.w[f"{name}.weight"]
        return y + self.w[f"{name}.bias"] if bias else y

    def linear(self, x, name, bias=True):
        y = x @ self.w[f"{name}.weight"].T
        return y + self.w[f"{name}.bias"] if bias else y

    # ---- tower ----------------------------------------------------------------------------
    def subsample(self, feats):
        """feats (1, F, 128) -> (1, T, 1024), T = ceil(ceil(F / 2) / 2). NHWC throughout: (B, time, mel, channels)."""
        x = feats[..., None]
        for i in range(2):
            name = f"tower.subsample_conv_projection.layer{i}"
            x = mx.conv2d(x, self.w[f"{name}.conv.weight"], stride=(2, 2), padding=(1, 1))
            x = mx.maximum(self.layer_norm(x, f"{name}.norm", eps=RMS_EPS, bias=False), 0.0)
        x = x.reshape(x.shape[0], x.shape[1], -1)
        return self.linear(x, "tower.subsample_conv_projection.input_proj_linear", bias=False)

    def feed_forward(self, x, name):
        res = x
        x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP)
        x = self.rms(x, f"{name}.pre_layer_norm")
        x = silu(self.clipped(x, f"{name}.ffw_layer_1"))
        x = self.clipped(x, f"{name}.ffw_layer_2")
        x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP)
        x = self.rms(x, f"{name}.post_layer_norm")
        return x * 0.5 + res

    @staticmethod
    def blocked_mask(T: int) -> mx.array:
        """True = blocked, shape (blocks, CHUNK, CONTEXT): a frame sees itself and the 11 frames before it, valid frames only.
        The same as transformers' sliding window mask (left window attention_context_left - 1 = 12, dist < 12) in blocked form."""
        blocks = (T + CHUNK - 1) // CHUNK
        q = mx.arange(blocks * CHUNK).reshape(blocks, CHUNK, 1)
        start = (mx.arange(blocks) * CHUNK)[:, None, None]
        kv = start - PAST + mx.arange(CONTEXT)[None, None, :]
        dist = q - kv
        allowed = (dist >= 0) & (dist < PAST) & (kv >= 0) & (kv < T) & (q < T)
        return ~allowed

    def attention(self, x, name, mask):
        B, T, _ = x.shape
        shape = (B, T, HEADS, HEAD_DIM)
        q = self.clipped(x, f"{name}.q_proj").reshape(shape)
        k = self.clipped(x, f"{name}.k_proj").reshape(shape)
        v = self.clipped(x, f"{name}.v_proj").reshape(shape)
        q = q * Q_SCALE * softplus(self.w[f"{name}.per_dim_scale"])
        k = k * K_SCALE

        blocks = (T + CHUNK - 1) // CHUNK
        pad = blocks * CHUNK - T
        q = mx.pad(q, [(0, 0), (0, pad), (0, 0), (0, 0)]).reshape(B, blocks, CHUNK, HEADS, HEAD_DIM)
        idx = (mx.arange(blocks) * CHUNK)[:, None] + mx.arange(CONTEXT)[None, :]
        pad_ctx = [(0, 0), (PAST, CHUNK - 1), (0, 0), (0, 0)]
        k = mx.pad(k, pad_ctx)[:, idx]  # (B, blocks, CONTEXT, HEADS, HEAD_DIM)
        v = mx.pad(v, pad_ctx)[:, idx]

        rel = (self.pos @ self.w[f"{name}.relative_k_proj.weight"].T).reshape(-1, HEADS, HEAD_DIM)  # (13, H, D)

        queries = q.transpose(0, 3, 1, 2, 4)  # (B, H, blocks, CHUNK, D)
        ac = queries @ k.transpose(0, 3, 1, 4, 2)  # (B, H, blocks, CHUNK, CONTEXT)
        flat = queries.reshape(B, HEADS, -1, HEAD_DIM)
        bd = (flat @ rel.transpose(1, 2, 0)).reshape(B, HEADS, blocks, CHUNK, rel.shape[0])
        # _rel_shift: pad positions to CONTEXT + 1, flatten rows, cut, reshape; each row moves one place left.
        bd = mx.pad(bd, [(0, 0), (0, 0), (0, 0), (0, 0), (0, CONTEXT + 1 - rel.shape[0])])
        bd = bd.reshape(B, HEADS, blocks, CHUNK * (CONTEXT + 1))[..., : CHUNK * CONTEXT]
        bd = bd.reshape(B, HEADS, blocks, CHUNK, CONTEXT)

        weights = mx.tanh((ac + bd) / SOFTCAP) * SOFTCAP
        weights = mx.where(mask, INVALID, weights)
        weights = softmax(weights)
        out = weights @ v.transpose(0, 3, 1, 2, 4)  # (B, H, blocks, CHUNK, D)
        out = out.transpose(0, 2, 3, 1, 4).reshape(B, blocks * CHUNK, -1)[:, :T]
        return self.clipped(out, f"{name}.post")

    def light_conv(self, x, name):
        res = x
        x = self.rms(x, f"{name}.pre_layer_norm")
        x = self.clipped(x, f"{name}.linear_start")
        a, b = mx.split(x, 2, axis=-1)  # GLU
        x = a * mx.sigmoid(b)
        x = mx.pad(x, [(0, 0), (4, 0), (0, 0)])  # causal, kernel 5
        x = mx.conv1d(x, self.w[f"{name}.depthwise_conv1d.weight"], groups=HID)
        x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP)
        x = self.rms(x, f"{name}.conv_norm")
        x = silu(x)
        return self.clipped(x, f"{name}.linear_end") + res

    def layer(self, x, i, mask):
        name = f"tower.layers.{i}"
        x = self.feed_forward(x, f"{name}.feed_forward1")
        res = x
        x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP)
        x = self.rms(x, f"{name}.norm_pre_attn")
        x = self.attention(x, f"{name}.self_attn", mask)
        x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP)
        x = self.rms(x, f"{name}.norm_post_attn") + res
        x = self.light_conv(x, f"{name}.lconv1d")
        x = self.feed_forward(x, f"{name}.feed_forward2")
        x = mx.clip(x, -GRAD_CLIP, GRAD_CLIP)
        return self.rms(x, f"{name}.norm_out")

    # ---- reader ---------------------------------------------------------------------------
    def reader_block(self, x, i, block, stack: str = ""):
        name = f"{stack}reader.layers.{i}"
        B, T, D = x.shape
        y = self.layer_norm(x, f"{name}.norm1")
        qkv = y @ self.w[f"{name}.self_attn.in_proj_weight"].T + self.w[f"{name}.self_attn.in_proj_bias"]
        d = D // READER_HEADS
        q, k, v = [t.reshape(B, T, READER_HEADS, d).transpose(0, 2, 1, 3) for t in mx.split(qkv, 3, axis=-1)]
        a = (q @ k.transpose(0, 1, 3, 2)) / math.sqrt(d)
        if block is not None:
            a = mx.where(block, -mx.inf, a)
        y = (softmax(a) @ v).transpose(0, 2, 1, 3).reshape(B, T, D)
        x = x + self.linear(y, f"{name}.self_attn.out_proj")
        y = self.layer_norm(x, f"{name}.norm2")
        return x + self.linear(gelu(self.linear(y, f"{name}.linear1")), f"{name}.linear2")

    def encode(self, feats, streaming: bool = False, stack: str = ""):
        """feats (1, F, 128) -> normalized reader output (1, T, 768). stack: the post-tower stack's state-dict prefix, "" (the
        Tamil stack, both modes of a file without the fork) or FORK (STT-2's code-mix reader fork); the tower is the same."""
        hs = self.subsample(feats)
        T = hs.shape[1]
        mask = self.blocked_mask(T)
        weights = softmax(self.w[f"{stack}layer_logits"])
        mixed = weights[0] * hs
        for i in range(TOWER_LAYERS):
            hs = self.layer(hs, i, mask)
            mixed = mixed + weights[i + 1] * hs
            mx.eval(hs, mixed)  # keep the lazy graph small
        x = self.linear(self.layer_norm(mixed, f"{stack}in_norm"), f"{stack}in_proj")
        y = mx.pad(x, [(0, 0), (14, 0), (0, 0)])
        y = mx.conv1d(y, self.w[f"{stack}position.conv.weight"], groups=READER_DIM) + self.w[f"{stack}position.conv.bias"]
        x = x + gelu(y)
        block = None
        if streaming:
            idx = mx.arange(T) // STREAM_CHUNK
            block = idx[None, :] > idx[:, None]  # True = may not attend
        for i in range(READER_LAYERS):
            x = self.reader_block(x, i, block, stack)
        x = self.layer_norm(x, f"{stack}out_norm")
        mx.eval(x)
        return x

    # ---- heads ----------------------------------------------------------------------------
    def ctc_logits(self, enc, head: str = "ctc_head"):
        return self.linear(enc, head)

    def ctc_log_probs(self, enc, head: str = "ctc_head"):
        """(1, T, V) float32 log-softmax, as model.ctc_log_probs."""
        logits = self.ctc_logits(enc, head)
        return logits - mx.logsumexp(logits, axis=-1, keepdims=True)

    def _lstm(self, x, h, c, head: str = "transducer"):
        w = self.w
        gates = (x @ w[f"{head}.lstm.weight_ih_l0"].T + w[f"{head}.lstm.bias_ih_l0"]
                 + h @ w[f"{head}.lstm.weight_hh_l0"].T + w[f"{head}.lstm.bias_hh_l0"])
        i, f, g, o = mx.split(gates, 4, axis=-1)
        c = mx.sigmoid(f) * c + mx.sigmoid(i) * mx.tanh(g)
        h = mx.sigmoid(o) * mx.tanh(c)
        return h, c

    def _predict(self, token: int, h, c, head: str = "transducer"):
        h, c = self._lstm(self.w[f"{head}.embed.weight"][token], h, c, head)
        return self.linear(h, f"{head}.pred_proj"), h, c

    def rnnt_greedy(self, enc, head: str = "transducer", start_penalty: float = 0.0, cue=None) -> list[int]:
        """Transducer.greedy for one clip: each frame emits tokens until blank, at most MAX_SYMBOLS. start_penalty is taken off
        blank's logit while nothing has been emitted, on the frames where cue (T,) bool is True (every frame if cue is None)."""
        proj = self.linear(enc[0], f"{head}.enc_proj")
        mx.eval(proj)
        h = c = mx.zeros((PRED,))
        pred, h, c = self._predict(0, h, c, head)
        out = []
        for t in range(proj.shape[0]):
            e = proj[t]
            for _ in range(MAX_SYMBOLS):
                logits = self.linear(mx.tanh(e + pred), f"{head}.out")
                if start_penalty and not out and (cue is None or cue[t]):
                    logits = mx.concatenate([logits[:1] - start_penalty, logits[1:]])
                k = mx.argmax(logits)
                mx.eval(k)
                k = int(k.item())
                if k == 0:
                    break
                pred, h, c = self._predict(k, h, c, head)
                mx.eval(pred, h, c)
                out.append(k)
        return out

    # ---- J(l, b): CTC-assisted rescoring (scripts/decode_ctc_assist.py), one clip ---------------
    def beam_decode(self, enc, head: str = "transducer", beam: int = 4, topk: int = 8, prune: float = 12.0):
        """rnnt_beam.beam_decode with no LM, bonus or mask, for one clip's reader output (T, D) -> all final hypotheses, best
        first, as (score, tokens). A hypothesis is [tokens, score, pred, h, c]; merging, pruning and order as there."""
        proj = self.linear(enc, f"{head}.enc_proj")
        pred, h, c = self._predict(0, mx.zeros((PRED,)), mx.zeros((PRED,)), head)
        mx.eval(proj, pred, h, c)
        hyps = [[(), 0.0, pred, h, c]]
        k_top = min(topk, self.w[f"{head}.out.weight"].shape[0] - 1)
        for t in range(proj.shape[0]):
            logits = self.linear(mx.tanh(proj[t][None] + mx.stack([x[2] for x in hyps])), f"{head}.out")
            logp = np.array(logits - mx.logsumexp(logits, axis=-1, keepdims=True))  # (H, V) float32
            tokens = np.argsort(-logp[:, 1:], axis=-1, kind="stable")[:, :k_top]
            values = np.take_along_axis(logp[:, 1:], tokens, axis=-1)
            blanks, values, tokens = logp[:, 0].tolist(), values.tolist(), tokens.tolist()
            cands = {x[0]: [x[1] + b, x, None] for x, b in zip(hyps, blanks)}
            for x, b, vs, ks in zip(hyps, blanks, values, tokens):
                floor = max(b, vs[0]) - prune
                for v, k in zip(vs, ks):
                    if v < floor or v == -math.inf:
                        break  # sorted
                    k += 1
                    score = x[1] + v
                    ys = x[0] + (k,)
                    seen = cands.get(ys)
                    if seen is None:
                        cands[ys] = [score, x, k]
                    else:  # an existing hypothesis that took blank here
                        seen[0] = _log_add(seen[0], score)
            top = heapq.nlargest(beam, cands.items(), key=lambda kv: kv[1][0])
            new = []
            for ys, (score, parent, token) in top:
                if token is None:
                    parent[1] = score
                    new.append(parent)
                else:
                    p, hn, cn = self._predict(token, parent[3], parent[4], head)
                    mx.eval(p, hn, cn)
                    new.append([ys, score, p, hn, cn])
            hyps = new
        final = [(x[1], x[0]) for x in hyps]
        final.sort(key=lambda r: -r[0])
        return final

    def rnnt_forced(self, proj, y: list[int], head: str = "transducer", cells: int = JOINT_CELLS) -> float:
        """decode_ctc_assist.rnnt_forced: log P_RNNT(y | x) over all alignments. proj = enc_proj(enc), (T, J). Joint float32,
        log-softmax and the lattice recursion float64."""
        T, U = proj.shape[0], len(y)
        h = c = mx.zeros((PRED,))
        preds = []
        for k in [0] + list(y):
            p, h, c = self._predict(k, h, c, head)
            preds.append(p)
        pred = mx.stack(preds)  # (U+1, J)
        blank = np.empty((T, U + 1), dtype=np.float64)
        emit = np.empty((T, U), dtype=np.float64)
        step = max(1, cells // (U + 1))
        for t0 in range(0, T, step):
            lg = np.array(self.linear(mx.tanh(proj[t0 : t0 + step, None] + pred[None]), f"{head}.out")).astype(np.float64)
            m = lg.max(-1, keepdims=True)
            lg = lg - (m + np.log(np.exp(lg - m).sum(-1, keepdims=True)))
            blank[t0 : t0 + step] = lg[..., 0]
            if U:
                emit[t0 : t0 + step] = lg[:, np.arange(U), np.asarray(y)]
        alpha = np.full(U + 1, -np.inf)
        alpha[0] = 0.0
        for t in range(T):
            a = alpha if t == 0 else alpha + blank[t - 1]  # from (t-1, u) by blank
            cs = np.concatenate([[0.0], np.cumsum(emit[t])])  # tokens within frame t
            alpha = cs + np.logaddexp.accumulate(a - cs)
        return float(alpha[U] + blank[T - 1, U])

    def joint_rescore(self, enc, head: str = "transducer", ctc: str = "ctc_head", lam: float = 0.3, beta: float = 0.0,
                      beam: int = 4) -> tuple[list[int], list]:
        """decode_ctc_assist.decode_clip's J(lam, beta) for one clip's reader output (T, D) -> (pick, scored candidates
        [(source, tokens, log P_RNNT, log P_CTC)]). Ties keep the earlier candidate (beam order, then CTC)."""
        logits = self.linear(enc, ctc)
        logp = np.array(logits - mx.logsumexp(logits, axis=-1, keepdims=True))  # (T, V) float32
        c_toks = ctc_greedy(logp.argmax(-1))
        nbest = self.beam_decode(enc, head, beam)
        cands, seen = [], set()
        for src, ys in [("beam", tuple(y)) for _, y in nbest] + [("ctc", tuple(c_toks))]:
            if ys not in seen:
                seen.add(ys)
                cands.append((src, list(ys)))
        proj = self.linear(enc, f"{head}.enc_proj")
        mx.eval(proj)
        scored = [(src, ys, self.rnnt_forced(proj, ys, head), ctc_forced(logp, ys)) for src, ys in cands]
        key = lambda s: (1 - lam) * s[2] + lam * s[3] + beta * len(s[1])  # noqa: E731
        return max(scored, key=key)[1], scored


def _log_add(a: float, b: float) -> float:
    return (a + math.log1p(math.exp(b - a))) if a >= b else (b + math.log1p(math.exp(a - b)))


def ctc_greedy(best) -> list[int]:
    """Frame argmax (T,) -> tokens: merge repeats, drop blanks (CTCTokenizer.greedy's rule)."""
    toks, prev = [], -1
    for k in np.asarray(best).tolist():
        if k != prev and k != 0:
            toks.append(k)
        prev = k
    return toks


def ctc_forced(logp: np.ndarray, y: list[int]) -> float:
    """decode_ctc_assist.ctc_forced: log P_CTC(y | x) over all alignments, the forward recursion in float64 on (T, V)
    log-probs, blank 0. -inf if y cannot fit in T frames."""
    T = logp.shape[0]
    if not y:
        return float(logp[:, 0].astype(np.float64).sum())
    ext = np.zeros(2 * len(y) + 1, dtype=np.int64)
    ext[1::2] = y
    lp = logp[:, ext].astype(np.float64)  # (T, S)
    skip = np.zeros(len(ext), dtype=bool)  # s-2 -> s: a token state whose token differs from the previous one
    skip[3::2] = ext[3::2] != ext[1:-2:2]
    alpha = np.full(len(ext), -np.inf)
    alpha[:2] = lp[0, :2]
    for t in range(1, T):
        a1 = np.concatenate([[-np.inf], alpha[:-1]])
        a2 = np.where(skip, np.concatenate([[-np.inf, -np.inf], alpha[:-2]]), -np.inf)
        alpha = np.logaddexp(np.logaddexp(alpha, a1), a2) + lp[t]
    return float(np.logaddexp(alpha[-1], alpha[-2]))


Stage6 = IndicSTT  # the stage 6 release's name for the same class