Spaces:
Running
Running
File size: 3,867 Bytes
9b3bb37 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | """Find where each word of a known transcript is spoken (CTC forced alignment over letters)."""
from __future__ import annotations
import re
from dataclasses import dataclass
import numpy as np
import torch
ALIGNER_MODEL = "facebook/wav2vec2-base-960h"
SECONDS_PER_FRAME = 320 / 16000
@dataclass
class WordSpan:
word: str
start: float # seconds
end: float
score: float # mean letter posterior, a rough alignment confidence
def words_of(transcript: str) -> list[str]:
return [w for w in re.sub(r"[^a-z' ]+", " ", transcript.lower().replace("’", "'")).split() if re.search(r"[a-z]", w)]
class WordAligner:
def __init__(self, model_id: str = ALIGNER_MODEL):
from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
self.processor = Wav2Vec2Processor.from_pretrained(model_id)
self.model = Wav2Vec2ForCTC.from_pretrained(model_id).eval()
self.vocab = self.processor.tokenizer.get_vocab()
self.blank = self.processor.tokenizer.pad_token_id
self.upper = all(k.isupper() for k in self.vocab if len(k) == 1 and k.isalpha())
@torch.no_grad()
def emissions(self, audio: np.ndarray, window_s: float = 20.0, overlap_s: float = 1.0) -> torch.Tensor:
"""Frame log-posteriors; long audio is processed in overlapping windows to bound memory."""
win, hop = int(window_s * 16000), int((window_s - overlap_s) * 16000)
if len(audio) <= win:
x = self.processor(audio, sampling_rate=16000, return_tensors="pt").input_values
return torch.log_softmax(self.model(x).logits[0], -1)
# place every window's frames by its sample offset (320 samples per frame) rather than by counting,
# because a 20 s window yields 999 frames, not 1000; overlaps are split at their midpoint
parts, written, half = [], 0, int(overlap_s / 2 / SECONDS_PER_FRAME)
for start in range(0, len(audio), hop):
chunk = audio[start:start + win]
x = self.processor(chunk, sampling_rate=16000, return_tensors="pt").input_values
lp = torch.log_softmax(self.model(x).logits[0], -1)
base = start // 320
last = start + win >= len(audio)
t0 = max(0, written - base)
t1 = len(lp) if last else len(lp) - half
if t1 > t0:
parts.append(lp[t0:t1])
written = base + t1
if last:
break
return torch.cat(parts)
def align(self, audio: np.ndarray, transcript: str) -> list[WordSpan]:
import torchaudio.functional as AF
words = words_of(transcript)
if not words:
return []
delim = self.vocab.get("|")
ids, owner = [], []
for wi, w in enumerate(words):
for ch in (w.upper() if self.upper else w):
if ch in self.vocab:
ids.append(self.vocab[ch])
owner.append(wi)
if delim is not None and wi < len(words) - 1:
ids.append(delim)
owner.append(-1)
lp = self.emissions(audio)
if len(ids) > len(lp):
raise ValueError("the transcript is longer than the audio can hold")
labels, scores = AF.forced_align(lp.unsqueeze(0), torch.tensor([ids], dtype=torch.int32), blank=self.blank)
spans = AF.merge_tokens(labels[0], scores[0].exp())
if len(spans) != len(ids):
raise RuntimeError("alignment lost letters")
out = []
for wi, w in enumerate(words):
mine = [s for s, o in zip(spans, owner) if o == wi]
if not mine:
continue
out.append(WordSpan(w, mine[0].start * SECONDS_PER_FRAME, mine[-1].end * SECONDS_PER_FRAME,
float(np.mean([s.score for s in mine]))))
return out
|