nida-tts / src /nida /data /workers /ctc_emissions.py
Bilal7Dev's picture
Nida v0.5 (research preview): weights, 16 voices, code, model card
c0bce59 verified
Raw History Blame Contribute Delete
8.38 kB
"""Turkish CTC emissions for forced alignment (stage S5).
Model: ``mpoyraz/wav2vec2-xls-r-300m-cv7-turkish`` at revision
``708639f50559d7970f462e13ec64d3f059ca89f6`` (CC-BY-4.0, CV7 tr test WER 8.62 / CER 2.26 per
the card). The alignment itself (Viterbi) lives in ``nida.data.align``; this worker only
produces the emissions.
Output per request: float32 ``[T, V]`` **log-softmax** emissions (``V`` = 38, the vocabulary
size; computed in float32) written to ``params["out_path"]``. Payload: ``frame_hop_ms``
(20.0, derived from the model's feature-extractor strides: ``prod(conv_stride) / 16000``
= 320 / 16000), ``blank_id`` (``config.pad_token_id``, the CTC blank ``[PAD]``) and ``vocab``
(token per id; NFC) so every result is self-describing.
APIs relied upon (transformers 4.57.6, the version pinned for Phase 0 and in
:mod:`nida.data.envs`):
* ``Wav2Vec2ForCTC.from_pretrained(path)`` -> ``model(input_values).logits`` of shape
``(batch, T, V)`` (https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2ForCTC).
* ``Wav2Vec2FeatureExtractor.from_pretrained(path)(audio, sampling_rate=16000,
return_tensors="pt")["input_values"]`` zero-mean/unit-variance normalises the waveform
(``do_normalize: true`` in the repo's ``preprocessor_config.json``).
* Deviation from "Wav2Vec2Processor": the repo's ``preprocessor_config.json`` declares
``processor_class: Wav2Vec2ProcessorWithLM`` (a pyctcdecode KenLM bundle we do not use), so
``Wav2Vec2Processor.from_pretrained`` is avoided. Only the feature extractor is needed and
the vocabulary is read straight from the repo's ``vocab.json`` (token -> id) and validated
against ``config.vocab_size``. Only ``*.json`` and ``pytorch_model.bin`` are downloaded (the
repo has no safetensors file and ships an unused ``language_model/lm.bin``).
Clips are processed one at a time: the model uses layer-norm feature extraction, so batching
needs attention masks and padding would perturb the emissions.
"""
from __future__ import annotations
import importlib
import json
import math
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import TYPE_CHECKING, Any
from nida.bench.protocol import WorkRequest
from nida.bench.workers.common import load_mono_16k, snapshot
from nida.bench.workers.runner import Adapter, ModelRef, main_for
from nida.data.workers import REQUIRED_SAMPLE_RATE_HZ, finish_frame_output, out_path_param
from nida.text.turkish import nfc
if TYPE_CHECKING:
import numpy.typing as npt
__all__ = [
"SNAPSHOT_PATTERNS",
"CtcEmissionsAdapter",
"load_vocab",
]
#: ``pytorch_model.bin`` is the only weight file in the repo; ``*.json`` brings the configs
#: and ``vocab.json`` (plus a few tiny files under ``language_model/``).
SNAPSHOT_PATTERNS: tuple[str, ...] = ("*.json", "pytorch_model.bin")
#: T = floor((n - 400) / 320) + 1 gives T * 20 ms in (duration - 25 ms, duration - 5 ms].
FRAME_TOLERANCE_FRAMES = 1.5
#: Rows of exp(log_softmax) must sum to 1 within this absolute error.
NORMALISATION_TOLERANCE = 1e-3
def load_vocab(path: Path) -> list[str]:
"""Read ``vocab.json`` (token -> id) into a list indexed by id (tokens NFC-normalised).
Raises:
ValueError: if the ids are not exactly ``0..V-1`` or tokens collide after NFC.
"""
data = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(data, dict) or not data:
raise ValueError(f"{path}: expected a non-empty JSON object token -> id")
tokens: list[str | None] = [None] * len(data)
for token, index in data.items():
if isinstance(index, bool) or not isinstance(index, int) or not 0 <= index < len(data):
raise ValueError(f"{path}: token {token!r} has invalid id {index!r}")
if tokens[index] is not None:
raise ValueError(f"{path}: id {index} is used by two tokens")
tokens[index] = nfc(token)
vocab = [t for t in tokens if t is not None]
if len(vocab) != len(data) or len(set(vocab)) != len(vocab):
raise ValueError(f"{path}: ids are not contiguous or tokens collide after NFC")
return vocab
class CtcEmissionsAdapter(Adapter):
"""wav2vec2 CTC log-softmax emissions; see the module docstring."""
name = "ctc_emissions"
distributions = ("torch", "transformers", "huggingface-hub", "soundfile", "numpy")
def __init__(self) -> None:
self._net: Any = None # Any: transformers Wav2Vec2ForCTC
self._extractor: Any = None # Any: transformers Wav2Vec2FeatureExtractor
self._torch: Any = None # Any: torch module, imported lazily
self._device = "cpu"
self._vocab: list[str] = []
self._blank_id = 0
self._frame_hop_ms = 0.0
def load(self, model: ModelRef, device: str, batch_size: int) -> None:
torch = importlib.import_module("torch")
transformers = importlib.import_module("transformers")
local = Path(snapshot(model.model_id, model.revision, SNAPSHOT_PATTERNS))
extractor = transformers.Wav2Vec2FeatureExtractor.from_pretrained(str(local))
if int(extractor.sampling_rate) != REQUIRED_SAMPLE_RATE_HZ:
raise ValueError(f"feature extractor expects {extractor.sampling_rate} Hz audio")
net = transformers.Wav2Vec2ForCTC.from_pretrained(str(local))
net = net.to(device=device, dtype=torch.float32).eval()
vocab = load_vocab(local / "vocab.json")
if len(vocab) != int(net.config.vocab_size):
raise ValueError(
f"vocab.json has {len(vocab)} tokens, model has {net.config.vocab_size}"
)
blank_id = int(net.config.pad_token_id)
if not 0 <= blank_id < len(vocab):
raise ValueError(f"CTC blank id {blank_id} is outside the vocabulary")
self._frame_hop_ms = 1000.0 * math.prod(int(s) for s in net.config.conv_stride)
self._frame_hop_ms /= REQUIRED_SAMPLE_RATE_HZ
self._net, self._extractor, self._torch = net, extractor, torch
self._device, self._vocab, self._blank_id = device, vocab, blank_id
def _emissions(self, audio: npt.NDArray[Any]) -> npt.NDArray[Any]:
"""Run the network: ``(samples,)`` at 16 kHz -> ``(T, V)`` float32 log-softmax."""
torch = self._torch
inputs = self._extractor(audio, sampling_rate=REQUIRED_SAMPLE_RATE_HZ, return_tensors="pt")
input_values = inputs["input_values"].to(self._device)
with torch.inference_mode():
logits = self._net(input_values).logits # (1, T, V)
log_probs = torch.log_softmax(logits.float(), dim=-1)
result: npt.NDArray[Any] = log_probs[0].cpu().numpy()
return result
def _emit(self, request: WorkRequest) -> Mapping[str, object]:
import numpy as np
out_path = out_path_param(request)
audio = load_mono_16k(request.audio_path)
duration_ms = 1000.0 * audio.shape[0] / REQUIRED_SAMPLE_RATE_HZ
emissions = np.asarray(self._emissions(audio), dtype=np.float32)
if emissions.ndim != 2 or emissions.shape[1] != len(self._vocab):
raise ValueError(
f"{request.clip_id}: emissions have shape {emissions.shape}, "
f"expected [T, {len(self._vocab)}]"
)
if not np.isfinite(emissions).all() or emissions.max() > 1e-4:
raise ValueError(f"{request.clip_id}: emissions are not finite log-probabilities")
row_sums = np.exp(emissions.astype(np.float64)).sum(axis=-1)
if np.abs(row_sums - 1.0).max() > NORMALISATION_TOLERANCE:
raise ValueError(f"{request.clip_id}: emission rows are not normalised log-softmax")
payload = finish_frame_output(
out_path=out_path,
array=emissions,
frame_hop_ms=self._frame_hop_ms,
duration_ms=duration_ms,
tolerance_ms=FRAME_TOLERANCE_FRAMES * self._frame_hop_ms,
what=f"{request.clip_id}: CTC emissions",
)
payload["blank_id"] = self._blank_id
payload["vocab"] = list(self._vocab)
payload["sample_rate_hz"] = REQUIRED_SAMPLE_RATE_HZ
return payload
def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]:
return [self._emit(request) for request in batch]
if __name__ == "__main__":
raise SystemExit(main_for(CtcEmissionsAdapter()))