"""Turkish CTC emissions for forced alignment (stage S5). Model: ``mpoyraz/wav2vec2-xls-r-300m-cv7-turkish`` at revision ``708639f50559d7970f462e13ec64d3f059ca89f6`` (CC-BY-4.0, CV7 tr test WER 8.62 / CER 2.26 per the card). The alignment itself (Viterbi) lives in ``nida.data.align``; this worker only produces the emissions. Output per request: float32 ``[T, V]`` **log-softmax** emissions (``V`` = 38, the vocabulary size; computed in float32) written to ``params["out_path"]``. Payload: ``frame_hop_ms`` (20.0, derived from the model's feature-extractor strides: ``prod(conv_stride) / 16000`` = 320 / 16000), ``blank_id`` (``config.pad_token_id``, the CTC blank ``[PAD]``) and ``vocab`` (token per id; NFC) so every result is self-describing. APIs relied upon (transformers 4.57.6, the version pinned for Phase 0 and in :mod:`nida.data.envs`): * ``Wav2Vec2ForCTC.from_pretrained(path)`` -> ``model(input_values).logits`` of shape ``(batch, T, V)`` (https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2ForCTC). * ``Wav2Vec2FeatureExtractor.from_pretrained(path)(audio, sampling_rate=16000, return_tensors="pt")["input_values"]`` zero-mean/unit-variance normalises the waveform (``do_normalize: true`` in the repo's ``preprocessor_config.json``). * Deviation from "Wav2Vec2Processor": the repo's ``preprocessor_config.json`` declares ``processor_class: Wav2Vec2ProcessorWithLM`` (a pyctcdecode KenLM bundle we do not use), so ``Wav2Vec2Processor.from_pretrained`` is avoided. Only the feature extractor is needed and the vocabulary is read straight from the repo's ``vocab.json`` (token -> id) and validated against ``config.vocab_size``. Only ``*.json`` and ``pytorch_model.bin`` are downloaded (the repo has no safetensors file and ships an unused ``language_model/lm.bin``). Clips are processed one at a time: the model uses layer-norm feature extraction, so batching needs attention masks and padding would perturb the emissions. """ from __future__ import annotations import importlib import json import math from collections.abc import Mapping, Sequence from pathlib import Path from typing import TYPE_CHECKING, Any from nida.bench.protocol import WorkRequest from nida.bench.workers.common import load_mono_16k, snapshot from nida.bench.workers.runner import Adapter, ModelRef, main_for from nida.data.workers import REQUIRED_SAMPLE_RATE_HZ, finish_frame_output, out_path_param from nida.text.turkish import nfc if TYPE_CHECKING: import numpy.typing as npt __all__ = [ "SNAPSHOT_PATTERNS", "CtcEmissionsAdapter", "load_vocab", ] #: ``pytorch_model.bin`` is the only weight file in the repo; ``*.json`` brings the configs #: and ``vocab.json`` (plus a few tiny files under ``language_model/``). SNAPSHOT_PATTERNS: tuple[str, ...] = ("*.json", "pytorch_model.bin") #: T = floor((n - 400) / 320) + 1 gives T * 20 ms in (duration - 25 ms, duration - 5 ms]. FRAME_TOLERANCE_FRAMES = 1.5 #: Rows of exp(log_softmax) must sum to 1 within this absolute error. NORMALISATION_TOLERANCE = 1e-3 def load_vocab(path: Path) -> list[str]: """Read ``vocab.json`` (token -> id) into a list indexed by id (tokens NFC-normalised). Raises: ValueError: if the ids are not exactly ``0..V-1`` or tokens collide after NFC. """ data = json.loads(path.read_text(encoding="utf-8")) if not isinstance(data, dict) or not data: raise ValueError(f"{path}: expected a non-empty JSON object token -> id") tokens: list[str | None] = [None] * len(data) for token, index in data.items(): if isinstance(index, bool) or not isinstance(index, int) or not 0 <= index < len(data): raise ValueError(f"{path}: token {token!r} has invalid id {index!r}") if tokens[index] is not None: raise ValueError(f"{path}: id {index} is used by two tokens") tokens[index] = nfc(token) vocab = [t for t in tokens if t is not None] if len(vocab) != len(data) or len(set(vocab)) != len(vocab): raise ValueError(f"{path}: ids are not contiguous or tokens collide after NFC") return vocab class CtcEmissionsAdapter(Adapter): """wav2vec2 CTC log-softmax emissions; see the module docstring.""" name = "ctc_emissions" distributions = ("torch", "transformers", "huggingface-hub", "soundfile", "numpy") def __init__(self) -> None: self._net: Any = None # Any: transformers Wav2Vec2ForCTC self._extractor: Any = None # Any: transformers Wav2Vec2FeatureExtractor self._torch: Any = None # Any: torch module, imported lazily self._device = "cpu" self._vocab: list[str] = [] self._blank_id = 0 self._frame_hop_ms = 0.0 def load(self, model: ModelRef, device: str, batch_size: int) -> None: torch = importlib.import_module("torch") transformers = importlib.import_module("transformers") local = Path(snapshot(model.model_id, model.revision, SNAPSHOT_PATTERNS)) extractor = transformers.Wav2Vec2FeatureExtractor.from_pretrained(str(local)) if int(extractor.sampling_rate) != REQUIRED_SAMPLE_RATE_HZ: raise ValueError(f"feature extractor expects {extractor.sampling_rate} Hz audio") net = transformers.Wav2Vec2ForCTC.from_pretrained(str(local)) net = net.to(device=device, dtype=torch.float32).eval() vocab = load_vocab(local / "vocab.json") if len(vocab) != int(net.config.vocab_size): raise ValueError( f"vocab.json has {len(vocab)} tokens, model has {net.config.vocab_size}" ) blank_id = int(net.config.pad_token_id) if not 0 <= blank_id < len(vocab): raise ValueError(f"CTC blank id {blank_id} is outside the vocabulary") self._frame_hop_ms = 1000.0 * math.prod(int(s) for s in net.config.conv_stride) self._frame_hop_ms /= REQUIRED_SAMPLE_RATE_HZ self._net, self._extractor, self._torch = net, extractor, torch self._device, self._vocab, self._blank_id = device, vocab, blank_id def _emissions(self, audio: npt.NDArray[Any]) -> npt.NDArray[Any]: """Run the network: ``(samples,)`` at 16 kHz -> ``(T, V)`` float32 log-softmax.""" torch = self._torch inputs = self._extractor(audio, sampling_rate=REQUIRED_SAMPLE_RATE_HZ, return_tensors="pt") input_values = inputs["input_values"].to(self._device) with torch.inference_mode(): logits = self._net(input_values).logits # (1, T, V) log_probs = torch.log_softmax(logits.float(), dim=-1) result: npt.NDArray[Any] = log_probs[0].cpu().numpy() return result def _emit(self, request: WorkRequest) -> Mapping[str, object]: import numpy as np out_path = out_path_param(request) audio = load_mono_16k(request.audio_path) duration_ms = 1000.0 * audio.shape[0] / REQUIRED_SAMPLE_RATE_HZ emissions = np.asarray(self._emissions(audio), dtype=np.float32) if emissions.ndim != 2 or emissions.shape[1] != len(self._vocab): raise ValueError( f"{request.clip_id}: emissions have shape {emissions.shape}, " f"expected [T, {len(self._vocab)}]" ) if not np.isfinite(emissions).all() or emissions.max() > 1e-4: raise ValueError(f"{request.clip_id}: emissions are not finite log-probabilities") row_sums = np.exp(emissions.astype(np.float64)).sum(axis=-1) if np.abs(row_sums - 1.0).max() > NORMALISATION_TOLERANCE: raise ValueError(f"{request.clip_id}: emission rows are not normalised log-softmax") payload = finish_frame_output( out_path=out_path, array=emissions, frame_hop_ms=self._frame_hop_ms, duration_ms=duration_ms, tolerance_ms=FRAME_TOLERANCE_FRAMES * self._frame_hop_ms, what=f"{request.clip_id}: CTC emissions", ) payload["blank_id"] = self._blank_id payload["vocab"] = list(self._vocab) payload["sample_rate_hz"] = REQUIRED_SAMPLE_RATE_HZ return payload def run_batch(self, batch: Sequence[WorkRequest]) -> list[Mapping[str, object]]: return [self._emit(request) for request in batch] if __name__ == "__main__": raise SystemExit(main_for(CtcEmissionsAdapter()))