Download src/nida/data/workers/ctc_emissions.py from Bilal7Dev/nida-tts: direct link, hf CLI and curl.
- Browser
- Download file 8.38 kB
-
https://huggingface.co/Bilal7Dev/nida-tts/resolve/main/src/nida/data/workers/ctc_emissions.py
- Command line
-
hf download hf://Bilal7Dev/nida-tts/src/nida/data/workers/ctc_emissions.py
-
curl -L -o ctc_emissions.py https://huggingface.co/Bilal7Dev/nida-tts/resolve/main/src/nida/data/workers/ctc_emissions.py
8.38 kB
| """Turkish CTC emissions for forced alignment (stage S5). | |
| Model: ``mpoyraz/wav2vec2-xls-r-300m-cv7-turkish`` at revision | |
| ``708639f50559d7970f462e13ec64d3f059ca89f6`` (CC-BY-4.0, CV7 tr test WER 8.62 / CER 2.26 per | |
| the card). The alignment itself (Viterbi) lives in ``nida.data.align``; this worker only | |
| produces the emissions. | |
| Output per request: float32 ``[T, V]`` **log-softmax** emissions (``V`` = 38, the vocabulary | |
| size; computed in float32) written to ``params["out_path"]``. Payload: ``frame_hop_ms`` | |
| (20.0, derived from the model's feature-extractor strides: ``prod(conv_stride) / 16000`` | |
| = 320 / 16000), ``blank_id`` (``config.pad_token_id``, the CTC blank ``[PAD]``) and ``vocab`` | |
| (token per id; NFC) so every result is self-describing. | |
| APIs relied upon (transformers 4.57.6, the version pinned for Phase 0 and in | |
| :mod:`nida.data.envs`): | |
| * ``Wav2Vec2ForCTC.from_pretrained(path)`` -> ``model(input_values).logits`` of shape | |
| ``(batch, T, V)`` (https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2ForCTC). | |
| * ``Wav2Vec2FeatureExtractor.from_pretrained(path)(audio, sampling_rate=16000, | |
| return_tensors="pt")["input_values"]`` zero-mean/unit-variance normalises the waveform | |
| (``do_normalize: true`` in the repo's ``preprocessor_config.json``). | |
| * Deviation from "Wav2Vec2Processor": the repo's ``preprocessor_config.json`` declares | |
| ``processor_class: Wav2Vec2ProcessorWithLM`` (a pyctcdecode KenLM bundle we do not use), so | |
| ``Wav2Vec2Processor.from_pretrained`` is avoided. Only the feature extractor is needed and | |
| the vocabulary is read straight from the repo's ``vocab.json`` (token -> id) and validated | |
| against ``config.vocab_size``. Only ``*.json`` and ``pytorch_model.bin`` are downloaded (the | |
| repo has no safetensors file and ships an unused ``language_model/lm.bin``). | |
| Clips are processed one at a time: the model uses layer-norm feature extraction, so batching | |
| needs attention masks and padding would perturb the emissions. | |
| """ | |
| from __future__ import annotations | |
| import importlib | |
| import json | |
| import math | |
| from collections.abc import Mapping, Sequence | |
| from pathlib import Path | |
| from typing import TYPE_CHECKING, Any | |
| from nida.bench.protocol import WorkRequest | |
| from nida.bench.workers.common import load_mono_16k, snapshot | |
| from nida.bench.workers.runner import Adapter, ModelRef, main_for | |
| from nida.data.workers import REQUIRED_SAMPLE_RATE_HZ, finish_frame_output, out_path_param | |
| from nida.text.turkish import nfc | |
| if TYPE_CHECKING: | |
| import numpy.typing as npt | |
| __all__ = [ | |
| "SNAPSHOT_PATTERNS", | |
| "CtcEmissionsAdapter", | |
| "load_vocab", | |
| ] | |
| #: ``pytorch_model.bin`` is the only weight file in the repo; ``*.json`` brings the configs | |
| #: and ``vocab.json`` (plus a few tiny files under ``language_model/``). | |
| SNAPSHOT_PATTERNS: tuple[str, ...] = ("*.json", "pytorch_model.bin") | |
| #: T = floor((n - 400) / 320) + 1 gives T * 20 ms in (duration - 25 ms, duration - 5 ms]. | |
| FRAME_TOLERANCE_FRAMES = 1.5 | |
| #: Rows of exp(log_softmax) must sum to 1 within this absolute error. | |
| NORMALISATION_TOLERANCE = 1e-3 | |
| def load_vocab(path: Path) -> list[str]: | |
| """Read ``vocab.json`` (token -> id) into a list indexed by id (tokens NFC-normalised). | |
| Raises: | |
| ValueError: if the ids are not exactly ``0..V-1`` or tokens collide after NFC. | |
| """ | |
| data = json.loads(path.read_text(encoding="utf-8")) | |
| if not isinstance(data, dict) or not data: | |
| raise ValueError(f"{path}: expected a non-empty JSON object token -> id") | |
| tokens: list[str | None] = [None] * len(data) | |
| for token, index in data.items(): | |
| if isinstance(index, bool) or not isinstance(index, int) or not 0 <= index < len(data): | |
| raise ValueError(f"{path}: token {token!r} has invalid id {index!r}") | |
| if tokens[index] is not None: | |
| raise ValueError(f"{path}: id {index} is used by two tokens") | |
| tokens[index] = nfc(token) | |
| vocab = [t for t in tokens if t is not None] | |
| if len(vocab) != len(data) or len(set(vocab)) != len(vocab): | |
| raise ValueError(f"{path}: ids are not contiguous or tokens collide after NFC") | |
| return vocab | |
| class CtcEmissionsAdapter(Adapter): | |
| """wav2vec2 CTC log-softmax emissions; see the module docstring.""" | |
| name = "ctc_emissions" | |
| distributions = ("torch", "transformers", "huggingface-hub", "soundfile", "numpy") | |
| def __init__(self) -> None: | |
| self._net: Any = None # Any: transformers Wav2Vec2ForCTC | |
| self._extractor: Any = None # Any: transformers Wav2Vec2FeatureExtractor | |
| self._torch: Any = None # Any: torch module, imported lazily | |
| self._device = "cpu" | |
| self._vocab: list[str] = [] | |
| self._blank_id = 0 | |
| self._frame_hop_ms = 0.0 | |
| def load(self, model: ModelRef, device: str, batch_size: int) -> None: | |
| torch = importlib.import_module("torch") | |
| transformers = importlib.import_module("transformers") | |
| local = Path(snapshot(model.model_id, model.revision, SNAPSHOT_PATTERNS)) | |
| extractor = transformers.Wav2Vec2FeatureExtractor.from_pretrained(str(local)) | |
| if int(extractor.sampling_rate) != REQUIRED_SAMPLE_RATE_HZ: | |
| raise ValueError(f"feature extractor expects {extractor.sampling_rate} Hz audio") | |
| net = transformers.Wav2Vec2ForCTC.from_pretrained(str(local)) | |
| net = net.to(device=device, dtype=torch.float32).eval() | |
| vocab = load_vocab(local / "vocab.json") | |
| if len(vocab) != int(net.config.vocab_size): | |
| raise ValueError( | |
| f"vocab.json has {len(vocab)} tokens, model has {net.config.vocab_size}" | |
| ) | |
| blank_id = int(net.config.pad_token_id) | |
| if not 0 <= blank_id < len(vocab): | |
| raise ValueError(f"CTC blank id {blank_id} is outside the vocabulary") | |
| self._frame_hop_ms = 1000.0 * math.prod(int(s) for s in net.config.conv_stride) | |
| self._frame_hop_ms /= REQUIRED_SAMPLE_RATE_HZ | |
| self._net, self._extractor, self._torch = net, extractor, torch | |
| self._device, self._vocab, self._blank_id = device, vocab, blank_id | |
| def _emissions(self, audio: npt.NDArray[Any]) -> npt.NDArray[Any]: | |
| """Run the network: ``(samples,)`` at 16 kHz -> ``(T, V)`` float32 log-softmax.""" | |
| torch = self._torch | |
| inputs = self._extractor(audio, sampling_rate=REQUIRED_SAMPLE_RATE_HZ, return_tensors="pt") | |
| input_values = inputs["input_values"].to(self._device) | |
| with torch.inference_mode(): | |
| logits = self._net(input_values).logits # (1, T, V) | |
| log_probs = torch.log_softmax(logits.float(), dim=-1) | |
| result: npt.NDArray[Any] = log_probs[0].cpu().numpy() | |
| return result | |
| def _emit(self, request: WorkRequest) -> Mapping[str, object]: | |
| import numpy as np | |
| out_path = out_path_param(request) | |
| audio = load_mono_16k(request.audio_path) | |
| duration_ms = 1000.0 * audio.shape[0] / REQUIRED_SAMPLE_RATE_HZ | |
| emissions = np.asarray(self._emissions(audio), dtype=np.float32) | |
| if emissions.ndim != 2 or emissions.shape[1] != len(self._vocab): | |
| raise ValueError( | |
| f"{request.clip_id}: emissions have shape {emissions.shape}, " | |
| f"expected [T, {len(self._vocab)}]" | |
| ) | |
| if not np.isfinite(emissions).all() or emissions.max() > 1e-4: | |
| raise ValueError(f"{request.clip_id}: emissions are not finite log-probabilities") | |
| row_sums = np.exp(emissions.astype(np.float64)).sum(axis=-1) | |
| if np.abs(row_sums - 1.0).max() > NORMALISATION_TOLERANCE: | |
| raise ValueError(f"{request.clip_id}: emission rows are not normalised log-softmax") | |
| payload = finish_frame_output( | |
| out_path=out_path, | |
| array=emissions, | |
| frame_hop_ms=self._frame_hop_ms, | |
| duration_ms=duration_ms, | |
| tolerance_ms=FRAME_TOLERANCE_FRAMES * self._frame_hop_ms, | |
| what=f"{request.clip_id}: CTC emissions", | |
| ) | |
| payload["blank_id"] = self._blank_id | |
| payload["vocab"] = list(self._vocab) | |
| payload["sample_rate_hz"] = REQUIRED_SAMPLE_RATE_HZ | |
| return payload | |
| def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]: | |
| return [self._emit(request) for request in batch] | |
| if __name__ == "__main__": | |
| raise SystemExit(main_for(CtcEmissionsAdapter())) | |