Spaces:
Running on Zero
Running on Zero
File size: 9,285 Bytes
e3a3acf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | """KenLM beam decoder for the CTC log-probabilities of balbal-v2 / balbal-v2-manas.
Input: natural-log CTC probabilities [frames, V + 1], blank last. Settings: configs/<model>/decoder.json (6-gram token KenLM, alpha,
beta, temperature, beam 64, top tokens 12, threshold 4, blank_collapse .99). Before the search the emissions are calibrated
(temperature, blank bias) and runs of confident blank frames (p(blank) > blank_collapse) are shortened to their last frame; word
timestamps are mapped back to the original frames.
Two searches with the same best path:
fast native/ctc_fast_decoder (default): flashlight's LexiconFreeDecoder search, but hypotheses are visited best first and a
token is skipped once even the largest possible LM score could not keep it in the beam, so fewer LM queries
flashlight flashlight-text's LexiconFreeDecoder itself (fast=False)
Optional hotwords (names/words expected in the audio) get a per-character bonus while decoding; see decode_words().
The LM directory holds model.binary (KenLM) and tokenizer.model (SentencePiece; piece i is the LM word chr(i + 100)).
"""
from __future__ import annotations
import math
from pathlib import Path
import sys
import numpy as np
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
from word_timing import token_spans, words_from_spans
def prepare(array, settings):
"""Calibrated emissions with confident-blank runs shortened, and the original frame index of every kept row."""
temperature, bias = settings.get('temperature', 1.), settings.get('blank_bias', 0.)
if temperature <= 0: raise ValueError('Positive temperature required')
a = np.ascontiguousarray(array, dtype=np.float32)
if temperature != 1 or bias != 0:
a = a / temperature
a[:, -1] += bias
maximum = a.max(axis=1, keepdims=True)
a = a - (maximum + np.log(np.exp(a - maximum).sum(axis=1, keepdims=True)))
kept = np.arange(len(a))
collapse = settings.get('blank_collapse')
if collapse is not None:
if not 0 < collapse < 1: raise ValueError('blank_collapse must be a probability')
confident = a[:, -1] > math.log(collapse)
# Keep the last blank of every confident run: the blank separating two repeated tokens is never dropped.
keep = np.ones(len(a), dtype=bool); keep[:-1] = ~(confident[:-1] & confident[1:])
a, kept = a[keep], kept[keep]
if not np.isfinite(a).all(): raise FloatingPointError('Nonfinite decoder input')
return np.ascontiguousarray(a, dtype=np.float32), kept
class KenLMDecoder:
"""CPU decoder; reuse one instance per thread/process."""
def __init__(self, lm_dir, settings, fast=True):
import sentencepiece as spm
from flashlight.lib.text.decoder import KenLM, ZeroLM
from flashlight.lib.text.dictionary import Dictionary
sys.path.insert(0, str(HERE / 'native'))
try:
from ctc_lm_bonus import HotwordLM, TokenBonusLM
if fast: from ctc_fast_decoder import FastCTCDecoder
except ImportError as error:
raise ImportError('Run python inference/build_native.py in this environment first.') from error
lm_dir = Path(lm_dir)
self.settings, self.fast = dict(settings), fast
s = self.settings
self.sp = spm.SentencePieceProcessor(model_file=str(lm_dir / 'tokenizer.model'))
self.vocab_size = self.sp.vocab_size()
self.blank_id = self.vocab_size
self.dictionary = Dictionary([chr(i + 100) for i in range(self.vocab_size)] + ['<unk>'])
self.lm = KenLM(str(lm_dir / 'model.binary'), self.dictionary) if s['alpha'] else ZeroLM()
self.scale = s['alpha'] if s['alpha'] else 1.0
self.bonus = s['beta'] / self.scale
self.wrapped_lm = TokenBonusLM(self.lm, self.bonus)
self._HotwordLM = HotwordLM
self._pieces = {self.sp.id_to_piece(i): i for i in range(self.vocab_size)}
self.decoder = self._make(self.wrapped_lm, self.bonus)
self._hot = (None, None) # (hotword list + bonus, its decoder): reused while the list stays the same
def _make(self, lm, upper):
"""A search over `lm`; upper = the largest score lm can return for a token (KenLM log10 <= 0, plus bonuses)."""
s = self.settings
if self.fast:
from ctc_fast_decoder import FastCTCDecoder
fd = FastCTCDecoder(int(s['beam']), int(s['top_tokens']), float(s['threshold']), self.scale, lm, self.blank_id, upper)
return lambda a: list(fd.decode(a.ctypes.data, *a.shape))
from flashlight.lib.text.decoder import CriterionType, LexiconFreeDecoder, LexiconFreeDecoderOptions
options = LexiconFreeDecoderOptions(int(s['beam']), int(s['top_tokens']), float(s['threshold']), self.scale, 0.0, True,
CriterionType.CTC)
decoder = LexiconFreeDecoder(options, lm, -1, self.blank_id, [])
def run(a):
hypotheses = decoder.decode(a.ctypes.data, *a.shape)
if not hypotheses: raise RuntimeError('Beam search returned no hypothesis')
return list(hypotheses[0].tokens)
run.keep = (lm, decoder)
return run
def hotword_sequences(self, words, bonus_per_char):
"""Token sequences for hotword biasing: every vocabulary segmentation (up to 64) of each word as written,
lowercased and capitalized (Turkish dotted/dotless i), so a word earns its bonus however the model spells it.
Each token's bonus is bonus_per_char x its characters (word-start marker included); phrases are split into words."""
lower = lambda w: w.replace('I', 'ı').replace('İ', 'i').lower()
upper1 = lambda w: (w[0].replace('i', 'İ').replace('ı', 'I').upper() + w[1:]) if w else w
seqs = set()
for phrase in words:
for word in str(phrase).split():
for variant in {word, lower(word), upper1(lower(word))}:
text, found = '▁' + variant, []
# ends[i]: text[i:] can be segmented; without it a character missing from the vocabulary makes the
# search below exponential in the word length (every prefix segmentation is tried and fails).
ends = [False] * len(text) + [True]
for i in range(len(text) - 1, -1, -1):
ends[i] = any(ends[j] and text[i:j] in self._pieces for j in range(i + 1, len(text) + 1))
def walk(i, path):
if len(found) >= 64:
return
if i == len(text):
found.append(tuple(path)); return
for j in range(len(text), i, -1):
piece = self._pieces.get(text[i:j])
if piece is not None and ends[j]:
walk(j, path + [piece])
walk(0, [])
seqs.update(found)
seqs = sorted(seqs)
bonuses = [[bonus_per_char * len(self.sp.id_to_piece(t)) / self.scale for t in q] for q in seqs]
return [list(q) for q in seqs], bonuses
def decode(self, log_probs, hotwords=None):
"""Best transcript for one utterance's [frames, V + 1] log-probabilities (see decode_words)."""
return self.decode_words(log_probs, hotwords)['text']
def decode_words(self, log_probs, hotwords=None, offset_seconds=0.):
"""Best transcript plus per-word timestamps: {'text', 'words': [{'word', 'start', 'end'}]} (seconds, shifted by offset_seconds).
hotwords: optional list of words/names expected in the audio (e.g. ['Eray', 'Hasan']). Tokens that extend a hotword earn
`hotword_bonus` (decoder.json) per character; the bonus is taken back if the word is not completed, so only whole hotwords
are rewarded."""
a = np.asarray(log_probs)
if a.ndim != 2 or a.shape[1] != self.vocab_size + 1 or not a.shape[0]:
raise ValueError(f'Expected nonempty [frames, {self.vocab_size + 1}] log-probabilities, blank last')
a, kept = prepare(a, self.settings)
decoder = self.decoder
if hotwords:
# A search over the shared KenLM per hotword list, kept for the next call with the same list (chunks of one file,
# streaming updates); the beam state resets on every decode.
key = tuple(hotwords), float(self.settings.get('hotword_bonus', 1.0))
hot = self._hot
if hot[0] != key:
seqs, bonuses = self.hotword_sequences(*key)
lm = self._HotwordLM(self.wrapped_lm, seqs, bonuses)
hot = self._hot = key, self._make(lm, self.bonus + max([max(b) for b in bonuses if b] + [0.]))
hot[1].keep_lm = lm
decoder = hot[1]
path = decoder(a)
# Collapse adjacent repeats before removing blanks: a blank separates repeated tokens. Frames back to the original grid.
spans = [(t, int(kept[f0]), int(kept[f1])) for t, f0, f1 in token_spans(path, self.vocab_size)]
return dict(text=self.sp.decode([t for t, _, _ in spans]), words=words_from_spans(spans, self.sp, offset_seconds))
|