"""--ctc-assist for transcribe.py: the J(l, b) CTC-assisted rescoring of the project's scripts/decode_ctc_assist.py, offline. Candidates: the RNN-T head's beam-4 n-best (modified beam search, no LM, no start penalty) and the CTC head's greedy hypothesis, duplicates merged. Each gets (1 - l) log P_RNNT(y) + l log P_CTC(y) + b |y| (|y| in tokens; both log P are exact forced scores summed over all alignments), and the best wins; ties keep the earlier candidate (beam order, then CTC). The release decodes with l = 0.3, b = 0 (transcribe.py LAMBDA, BETA), as `decode_ctc_assist.py --joint 0.3:0` ran. The functions below are verbatim copies (cut by source segment, not retyped) of the project's scripts/rnnt_beam.py sha256 5f83a87194f17e02478fd212d58a1588bf5a7f0a4f76f1b5d93276696e321766: Hyp, log_add, beam_decode scripts/decode_ctc_assist.py sha256 0d63a090992df8fd8e78d36be548306696adba727d6dd1c0a7baefb2bc8cfae1: JOINT_CELLS, ctc_forced, rnnt_forced, ctc_greedy joint_rescore() is decode_ctc_assist.decode_clip's J branch for one (l, b). The LM options of beam_decode are unused here. """ from __future__ import annotations import heapq import math import torch class Hyp: __slots__ = ("ys", "score", "pred", "state", "lm_state", "lm_next") def __init__(self, ys, score, pred, state, lm_state): self.ys, self.score, self.pred, self.state, self.lm_state = ys, score, pred, state, lm_state self.lm_next = {} # token -> (ln P, next LM state); a hypothesis stays in the beam over many blank frames def lm_step(self, lm: TokenLM, token: int): res = self.lm_next.get(token) if res is None: res = self.lm_next[token] = lm.step(self.lm_state, token) return res def log_add(a: float, b: float) -> float: return (a + math.log1p(math.exp(b - a))) if a >= b else (b + math.log1p(math.exp(a - b))) JOINT_CELLS = 8192 # T x (U+1) cells of one joint slice in rnnt_forced: 8192 x V x (4 + 8) bytes, about 200 MB at V = 2059 def ctc_forced(logp: torch.Tensor, y: list[int]) -> float: """log P_CTC(y | x), summed over all alignments: the CTC forward recursion in float64 on (T, V) log-probs, blank 0. -inf if y cannot fit in T frames. Equals -F.ctc_loss(reduction="sum") on the same log-probs.""" T = logp.shape[0] if not y: return float(logp[:, 0].double().sum()) ext = torch.zeros(2 * len(y) + 1, dtype=torch.long) ext[1::2] = torch.tensor(y) lp = logp[:, ext].double() # (T, S) skip = torch.zeros(len(ext), dtype=torch.bool) # s-2 -> s: a token state whose token differs from the previous one skip[3::2] = ext[3::2] != ext[1:-2:2] ninf = torch.tensor([-math.inf], dtype=torch.float64) alpha = torch.full((len(ext),), -math.inf, dtype=torch.float64) alpha[:2] = lp[0, :2] for t in range(1, T): a1 = torch.cat([ninf, alpha[:-1]]) a2 = torch.where(skip, torch.cat([ninf, ninf, alpha[:-2]]), ninf) alpha = torch.logsumexp(torch.stack([alpha, a1, a2]), 0) + lp[t] return float(torch.logaddexp(alpha[-1], alpha[-2])) @torch.no_grad() def rnnt_forced(head: Transducer, proj: torch.Tensor, y: list[int], style: int | None = None, cells: int = JOINT_CELLS) -> float: """log P_RNNT(y | x), summed over all alignments of the standard transducer lattice (the training loss's; any number of tokens per frame). proj = head.enc_proj(enc), (T, J). The joint is built a few frames at a time (<= cells of T x (U+1)) and only the blank and next-token log-probs are kept, so memory stays small. The log-softmax and the recursion (over u a log-cumsum-exp per frame) are float64 on the head's float32 logits: this equals a plain float64 lattice DP (measured <= 1e-6); torchaudio's float32 rnnt_loss is off by up to about 1e-3 absolute (measured on SPRING valid clips).""" T, U = proj.shape[0], len(y) tokens = torch.tensor([[0] + y]) pred, _ = head.predict(tokens) if style is None else head.predict(tokens, None, torch.full((1,), style)) pred = pred[0] # (U+1, J) blank = torch.empty(T, U + 1, dtype=torch.float64) emit = torch.empty(T, U, dtype=torch.float64) idx = torch.tensor(y)[None, :, None] step = max(1, cells // (U + 1)) for t0 in range(0, T, step): lg = head.joint(proj[t0 : t0 + step, None], pred[None]).double().log_softmax(-1) # (t, U+1, V) blank[t0 : t0 + step] = lg[..., 0] if U: emit[t0 : t0 + step] = lg[:, :U].gather(-1, idx.expand(lg.shape[0], U, 1))[..., 0] zero = torch.zeros(1, dtype=torch.float64) alpha = torch.full((U + 1,), -math.inf, dtype=torch.float64) alpha[0] = 0.0 for t in range(T): a = alpha if t == 0 else alpha + blank[t - 1] # from (t-1, u) by blank c = torch.cat([zero, emit[t].cumsum(0)]) # alpha(t, u) = c[u] + log sum_{k<=u} exp(a[k] - c[k]): tokens within frame t alpha = c + torch.logcumsumexp(a - c, 0) return float(alpha[U] + blank[T - 1, U]) def ctc_greedy(best: torch.Tensor) -> tuple[list[int], list[int]]: """Frame argmax (T,) -> tokens and each token's first frame: merge repeats, drop blanks (CTCTokenizer.greedy's rule).""" toks, frames, prev = [], [], -1 for t, k in enumerate(best.tolist()): if k != prev and k != 0: toks.append(k) frames.append(t) prev = k return toks, frames @torch.no_grad() def beam_decode(head, enc: torch.Tensor, lm: TokenLM | None = None, alpha: float = 0.0, beta: float = 0.0, beam: int = 4, topk: int = 8, prune: float = 12.0, blank_scoring: bool = False, bigram=None, gamma: float = 0.0, banned: torch.Tensor | None = None, swap: bool = False, first_bonus: float = 0.0, style: int | None = None) -> list[tuple[float, tuple[int, ...]]]: """Modified beam search over one clip's reader output (T, D) -> all final hypotheses, best first, as (score, tokens). Each frame extends each hypothesis by blank (same tokens, prediction and LM state, no bonus) or by one of its topk tokens, skipping tokens more than `prune` below that hypothesis' best RNN-T log-prob. The top `beam` survive after merging. Because the LM and bonus terms depend only on the tokens, merged paths add their RNN-T terms alone. Options (module docstring): blank_scoring needs the LM; bigram (V, V) of ln P, row 0 = sentence start; banned (V,) bool. """ def predict(tokens, states=None): return head.predict(tokens, states) if style is None else head.predict(tokens, states, torch.full((len(tokens),), style)) fuse = lm is not None and alpha != 0 blank_scale = 1.0 + alpha if fuse and blank_scoring else 1.0 lodr = gamma if bigram is not None else 0.0 proj = head.enc_proj(enc.float()) pred, state = predict(torch.zeros(1, 1, dtype=torch.long)) # start symbol = blank, as in training hyps = [Hyp((), 0.0, pred[0, 0], state, lm.begin() if fuse else None)] for frame in proj: logp = head.joint(frame[None], torch.stack([h.pred for h in hyps])).log_softmax(-1) must_emit = [False] * len(hyps) if banned is None: values, tokens = logp[:, 1:].topk(min(topk, logp.shape[1] - 1), dim=-1) else: if swap: must_emit = banned[logp.argmax(-1)].tolist() values, tokens = logp[:, 1:].masked_fill(banned[1:], -math.inf).topk(min(topk, logp.shape[1] - 1), dim=-1) blanks = logp[:, 0].tolist() cands = {h.ys: [h.score + blank_scale * b, h, None] for h, b, m in zip(hyps, blanks, must_emit) if not m} for h, b, m, vs, ks in zip(hyps, blanks, must_emit, values.tolist(), tokens.tolist()): floor = (vs[0] if m else max(b, vs[0])) - prune token_term = alpha * math.log(max(1.0 - math.exp(b), 1e-6)) if blank_scale != 1.0 else 0.0 for v, k in zip(vs, ks): if v < floor or v == -math.inf: # a banned token is -inf even with no pruning break # sorted k += 1 score = h.score + v + beta + token_term if first_bonus and not h.ys: score += first_bonus if fuse: score += alpha * h.lm_step(lm, k)[0] if lodr: score -= lodr * float(bigram[h.ys[-1] if h.ys else 0, k]) ys = h.ys + (k,) seen = cands.get(ys) if seen is None: cands[ys] = [score, h, k] else: # an existing hypothesis that took blank here: same tokens, so same LM, bonus and LODR terms seen[0] = log_add(seen[0], score) top = heapq.nlargest(beam, cands.items(), key=lambda kv: kv[1][0]) emitted = [c for _, c in top if c[2] is not None] if emitted: y, (hn, cn) = predict(torch.tensor([[c[2]] for c in emitted]), (torch.cat([c[1].state[0] for c in emitted], 1), torch.cat([c[1].state[1] for c in emitted], 1))) hyps, i = [], 0 for ys, (score, parent, token) in top: if token is None: parent.score = score hyps.append(parent) else: lm_state = parent.lm_step(lm, token)[1] if fuse else None hyps.append(Hyp(ys, score, y[i, 0], (hn[:, i : i + 1], cn[:, i : i + 1]), lm_state)) i += 1 final = [(h.score + (alpha * lm.end(h.lm_state) if fuse else 0.0), h.ys) for h in hyps] final.sort(key=lambda r: -r[0]) return final @torch.no_grad() def joint_rescore(head, ctc: torch.nn.Module, enc: torch.Tensor, lam: float, beta: float, beam: int = 4) -> tuple[list[int], list]: """One clip's reader output (T, D) -> the J(lam, beta) pick (model ids) and the scored candidates [(source, tokens, log P_RNNT, log P_CTC)]. head: a Transducer; ctc: the CTC head's Linear. decode_clip's J branch.""" enc = enc.float() logp = ctc(enc).float().log_softmax(-1) c_toks, _ = ctc_greedy(logp.argmax(-1)) nbest = beam_decode(head, enc, beam=beam, style=None) cands, seen = [], set() for src, ys in [("beam", tuple(y)) for _, y in nbest] + [("ctc", tuple(c_toks))]: if ys not in seen: seen.add(ys) cands.append((src, list(ys))) proj = head.enc_proj(enc) scored = [(src, ys, rnnt_forced(head, proj, ys, None), ctc_forced(logp, ys)) for src, ys in cands] key = lambda c: (1 - lam) * c[2] + lam * c[3] + beta * len(c[1]) # noqa: E731 return max(scored, key=key)[1], scored