VeriSafe-Tanglish-1-STT / code /ctc_assist.py
junafinity's picture
claude opus 5.5
VeriSafe-Tanglish-1-STT (STT-2)
ec01e2d
Raw History Blame Contribute Delete
10.6 kB
"""--ctc-assist for transcribe.py: the J(l, b) CTC-assisted rescoring of the project's scripts/decode_ctc_assist.py, offline.
Candidates: the RNN-T head's beam-4 n-best (modified beam search, no LM, no start penalty) and the CTC head's greedy
hypothesis, duplicates merged. Each gets (1 - l) log P_RNNT(y) + l log P_CTC(y) + b |y| (|y| in tokens; both log P are exact
forced scores summed over all alignments), and the best wins; ties keep the earlier candidate (beam order, then CTC). The
release decodes with l = 0.3, b = 0 (transcribe.py LAMBDA, BETA), as `decode_ctc_assist.py --joint 0.3:0` ran.
The functions below are verbatim copies (cut by source segment, not retyped) of the project's
scripts/rnnt_beam.py sha256 5f83a87194f17e02478fd212d58a1588bf5a7f0a4f76f1b5d93276696e321766: Hyp, log_add, beam_decode
scripts/decode_ctc_assist.py sha256 0d63a090992df8fd8e78d36be548306696adba727d6dd1c0a7baefb2bc8cfae1: JOINT_CELLS, ctc_forced, rnnt_forced, ctc_greedy
joint_rescore() is decode_ctc_assist.decode_clip's J branch for one (l, b). The LM options of beam_decode are unused here.
"""
from __future__ import annotations
import heapq
import math
import torch
class Hyp:
__slots__ = ("ys", "score", "pred", "state", "lm_state", "lm_next")
def __init__(self, ys, score, pred, state, lm_state):
self.ys, self.score, self.pred, self.state, self.lm_state = ys, score, pred, state, lm_state
self.lm_next = {} # token -> (ln P, next LM state); a hypothesis stays in the beam over many blank frames
def lm_step(self, lm: TokenLM, token: int):
res = self.lm_next.get(token)
if res is None:
res = self.lm_next[token] = lm.step(self.lm_state, token)
return res
def log_add(a: float, b: float) -> float:
return (a + math.log1p(math.exp(b - a))) if a >= b else (b + math.log1p(math.exp(a - b)))
JOINT_CELLS = 8192 # T x (U+1) cells of one joint slice in rnnt_forced: 8192 x V x (4 + 8) bytes, about 200 MB at V = 2059
def ctc_forced(logp: torch.Tensor, y: list[int]) -> float:
"""log P_CTC(y | x), summed over all alignments: the CTC forward recursion in float64 on (T, V) log-probs, blank 0.
-inf if y cannot fit in T frames. Equals -F.ctc_loss(reduction="sum") on the same log-probs."""
T = logp.shape[0]
if not y:
return float(logp[:, 0].double().sum())
ext = torch.zeros(2 * len(y) + 1, dtype=torch.long)
ext[1::2] = torch.tensor(y)
lp = logp[:, ext].double() # (T, S)
skip = torch.zeros(len(ext), dtype=torch.bool) # s-2 -> s: a token state whose token differs from the previous one
skip[3::2] = ext[3::2] != ext[1:-2:2]
ninf = torch.tensor([-math.inf], dtype=torch.float64)
alpha = torch.full((len(ext),), -math.inf, dtype=torch.float64)
alpha[:2] = lp[0, :2]
for t in range(1, T):
a1 = torch.cat([ninf, alpha[:-1]])
a2 = torch.where(skip, torch.cat([ninf, ninf, alpha[:-2]]), ninf)
alpha = torch.logsumexp(torch.stack([alpha, a1, a2]), 0) + lp[t]
return float(torch.logaddexp(alpha[-1], alpha[-2]))
@torch.no_grad()
def rnnt_forced(head: Transducer, proj: torch.Tensor, y: list[int], style: int | None = None, cells: int = JOINT_CELLS) -> float:
"""log P_RNNT(y | x), summed over all alignments of the standard transducer lattice (the training loss's; any number of
tokens per frame). proj = head.enc_proj(enc), (T, J). The joint is built a few frames at a time (<= cells of T x (U+1)) and
only the blank and next-token log-probs are kept, so memory stays small. The log-softmax and the recursion (over u a
log-cumsum-exp per frame) are float64 on the head's float32 logits: this equals a plain float64 lattice DP (measured
<= 1e-6); torchaudio's float32 rnnt_loss is off by up to about 1e-3 absolute (measured on SPRING valid clips)."""
T, U = proj.shape[0], len(y)
tokens = torch.tensor([[0] + y])
pred, _ = head.predict(tokens) if style is None else head.predict(tokens, None, torch.full((1,), style))
pred = pred[0] # (U+1, J)
blank = torch.empty(T, U + 1, dtype=torch.float64)
emit = torch.empty(T, U, dtype=torch.float64)
idx = torch.tensor(y)[None, :, None]
step = max(1, cells // (U + 1))
for t0 in range(0, T, step):
lg = head.joint(proj[t0 : t0 + step, None], pred[None]).double().log_softmax(-1) # (t, U+1, V)
blank[t0 : t0 + step] = lg[..., 0]
if U:
emit[t0 : t0 + step] = lg[:, :U].gather(-1, idx.expand(lg.shape[0], U, 1))[..., 0]
zero = torch.zeros(1, dtype=torch.float64)
alpha = torch.full((U + 1,), -math.inf, dtype=torch.float64)
alpha[0] = 0.0
for t in range(T):
a = alpha if t == 0 else alpha + blank[t - 1] # from (t-1, u) by blank
c = torch.cat([zero, emit[t].cumsum(0)]) # alpha(t, u) = c[u] + log sum_{k<=u} exp(a[k] - c[k]): tokens within frame t
alpha = c + torch.logcumsumexp(a - c, 0)
return float(alpha[U] + blank[T - 1, U])
def ctc_greedy(best: torch.Tensor) -> tuple[list[int], list[int]]:
"""Frame argmax (T,) -> tokens and each token's first frame: merge repeats, drop blanks (CTCTokenizer.greedy's rule)."""
toks, frames, prev = [], [], -1
for t, k in enumerate(best.tolist()):
if k != prev and k != 0:
toks.append(k)
frames.append(t)
prev = k
return toks, frames
@torch.no_grad()
def beam_decode(head, enc: torch.Tensor, lm: TokenLM | None = None, alpha: float = 0.0, beta: float = 0.0, beam: int = 4,
topk: int = 8, prune: float = 12.0, blank_scoring: bool = False, bigram=None, gamma: float = 0.0,
banned: torch.Tensor | None = None, swap: bool = False, first_bonus: float = 0.0,
style: int | None = None) -> list[tuple[float, tuple[int, ...]]]:
"""Modified beam search over one clip's reader output (T, D) -> all final hypotheses, best first, as (score, tokens).
Each frame extends each hypothesis by blank (same tokens, prediction and LM state, no bonus) or by one of its topk
tokens, skipping tokens more than `prune` below that hypothesis' best RNN-T log-prob. The top `beam` survive after
merging. Because the LM and bonus terms depend only on the tokens, merged paths add their RNN-T terms alone.
Options (module docstring): blank_scoring needs the LM; bigram (V, V) of ln P, row 0 = sentence start; banned (V,) bool.
"""
def predict(tokens, states=None):
return head.predict(tokens, states) if style is None else head.predict(tokens, states, torch.full((len(tokens),), style))
fuse = lm is not None and alpha != 0
blank_scale = 1.0 + alpha if fuse and blank_scoring else 1.0
lodr = gamma if bigram is not None else 0.0
proj = head.enc_proj(enc.float())
pred, state = predict(torch.zeros(1, 1, dtype=torch.long)) # start symbol = blank, as in training
hyps = [Hyp((), 0.0, pred[0, 0], state, lm.begin() if fuse else None)]
for frame in proj:
logp = head.joint(frame[None], torch.stack([h.pred for h in hyps])).log_softmax(-1)
must_emit = [False] * len(hyps)
if banned is None:
values, tokens = logp[:, 1:].topk(min(topk, logp.shape[1] - 1), dim=-1)
else:
if swap:
must_emit = banned[logp.argmax(-1)].tolist()
values, tokens = logp[:, 1:].masked_fill(banned[1:], -math.inf).topk(min(topk, logp.shape[1] - 1), dim=-1)
blanks = logp[:, 0].tolist()
cands = {h.ys: [h.score + blank_scale * b, h, None] for h, b, m in zip(hyps, blanks, must_emit) if not m}
for h, b, m, vs, ks in zip(hyps, blanks, must_emit, values.tolist(), tokens.tolist()):
floor = (vs[0] if m else max(b, vs[0])) - prune
token_term = alpha * math.log(max(1.0 - math.exp(b), 1e-6)) if blank_scale != 1.0 else 0.0
for v, k in zip(vs, ks):
if v < floor or v == -math.inf: # a banned token is -inf even with no pruning
break # sorted
k += 1
score = h.score + v + beta + token_term
if first_bonus and not h.ys:
score += first_bonus
if fuse:
score += alpha * h.lm_step(lm, k)[0]
if lodr:
score -= lodr * float(bigram[h.ys[-1] if h.ys else 0, k])
ys = h.ys + (k,)
seen = cands.get(ys)
if seen is None:
cands[ys] = [score, h, k]
else: # an existing hypothesis that took blank here: same tokens, so same LM, bonus and LODR terms
seen[0] = log_add(seen[0], score)
top = heapq.nlargest(beam, cands.items(), key=lambda kv: kv[1][0])
emitted = [c for _, c in top if c[2] is not None]
if emitted:
y, (hn, cn) = predict(torch.tensor([[c[2]] for c in emitted]),
(torch.cat([c[1].state[0] for c in emitted], 1),
torch.cat([c[1].state[1] for c in emitted], 1)))
hyps, i = [], 0
for ys, (score, parent, token) in top:
if token is None:
parent.score = score
hyps.append(parent)
else:
lm_state = parent.lm_step(lm, token)[1] if fuse else None
hyps.append(Hyp(ys, score, y[i, 0], (hn[:, i : i + 1], cn[:, i : i + 1]), lm_state))
i += 1
final = [(h.score + (alpha * lm.end(h.lm_state) if fuse else 0.0), h.ys) for h in hyps]
final.sort(key=lambda r: -r[0])
return final
@torch.no_grad()
def joint_rescore(head, ctc: torch.nn.Module, enc: torch.Tensor, lam: float, beta: float, beam: int = 4) -> tuple[list[int], list]:
"""One clip's reader output (T, D) -> the J(lam, beta) pick (model ids) and the scored candidates
[(source, tokens, log P_RNNT, log P_CTC)]. head: a Transducer; ctc: the CTC head's Linear. decode_clip's J branch."""
enc = enc.float()
logp = ctc(enc).float().log_softmax(-1)
c_toks, _ = ctc_greedy(logp.argmax(-1))
nbest = beam_decode(head, enc, beam=beam, style=None)
cands, seen = [], set()
for src, ys in [("beam", tuple(y)) for _, y in nbest] + [("ctc", tuple(c_toks))]:
if ys not in seen:
seen.add(ys)
cands.append((src, list(ys)))
proj = head.enc_proj(enc)
scored = [(src, ys, rnnt_forced(head, proj, ys, None), ctc_forced(logp, ys)) for src, ys in cands]
key = lambda c: (1 - lam) * c[2] + lam * c[3] + beta * len(c[1]) # noqa: E731
return max(scored, key=key)[1], scored