"""Processor that turns raw audio (and optional text) into model inputs.""" from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, ClassVar, cast, overload import numpy as np import numpy.typing as npt import torch import transformers from torch.nn.utils.rnn import pad_sequence from transformers import ( BatchFeature, PreTrainedTokenizerBase, ProcessorMixin, SequenceFeatureExtractor, ) if TYPE_CHECKING: from .asr_config import ( DEFAULT_ENCODER_CONV_LAYERS, ASRConfig, ConvLayerSpec, compute_encoder_output_length, ) from .asr_types import AudioFeatureExtractor, AudioInput, Waveform from .projectors import MLPAudioProjector else: try: from .asr_config import ( DEFAULT_ENCODER_CONV_LAYERS, ASRConfig, ConvLayerSpec, compute_encoder_output_length, ) from .asr_types import AudioInput except ImportError: # flat layout on the Hub: sibling modules, no package from asr_config import ( DEFAULT_ENCODER_CONV_LAYERS, ASRConfig, ConvLayerSpec, compute_encoder_output_length, ) from asr_types import AudioInput def left_pad_prompt_rows( rows: list[torch.Tensor], tokenizer: PreTrainedTokenizerBase ) -> tuple[torch.Tensor, torch.Tensor]: """Stack per-sample prompt rows into a left-padded batch: `(input_ids, attention_mask)`. Left, not right: these feed `generate`, so padding must not sit between the prompt and the first generated token. Pads with the tokenizer's pad token, falling back to eos, then 0. Pad positions never carry `audio_token_id`, so the model's masked_scatter is unaffected. """ # transformers types special-token ids as any token value; a single id is an int. pad_id = cast("int | None", tokenizer.pad_token_id) if pad_id is None: pad_id = cast("int | None", tokenizer.eos_token_id) or 0 input_ids = pad_sequence(rows, batch_first=True, padding_value=int(pad_id), padding_side="left") # Padded from ones rather than `input_ids != pad_id`: a real token may # equal `pad_id` when pad falls back to eos. attention_mask = pad_sequence( [torch.ones_like(row) for row in rows], batch_first=True, padding_side="left" ) return input_ids, attention_mask @overload def prepend_lead_in[ScalarT: np.generic]( audio: npt.NDArray[ScalarT], sampling_rate: int, seconds: float | None ) -> npt.NDArray[ScalarT]: ... @overload def prepend_lead_in[ScalarT: np.generic]( audio: list[npt.NDArray[ScalarT]], sampling_rate: int, seconds: float | None ) -> list[npt.NDArray[ScalarT]]: ... @overload def prepend_lead_in(audio: AudioInput, sampling_rate: int, seconds: float | None) -> AudioInput: ... def prepend_lead_in(audio: AudioInput, sampling_rate: int, seconds: float | None) -> AudioInput: """Prepend `seconds` of silence to a waveform (or each waveform in a list). Peoples ships fixed ~15s grid cuts rather than sentence-aligned segments, so a clip routinely opens mid-word and the model declines to emit the partial first token. Measured on 500 Peoples clips with a paired bootstrap: 20.51% -> 19.28% WER (delta -1.22, CI [-1.83, -0.64]) and utterances dropping a leading reference word fall 258/460 -> 170/460. CommonVoice, whose clips already start cleanly, is unaffected (+0.30, CI [-0.43, +1.17]). Inference only. Training feeds raw audio through the collator, so this is a test-time transform, and it recovers two thirds of the dropped onsets rather than all of them -- the remainder are clips whose first syllable was never recorded, which no amount of lead-in reconstructs. """ if not seconds or seconds <= 0: return audio if isinstance(audio, (list, tuple)) and audio and not isinstance(audio[0], (int, float)): batch = cast("Sequence[Waveform]", audio) return [cast("Waveform", prepend_lead_in(a, sampling_rate, seconds)) for a in batch] waveform = cast("Waveform", audio) pad = round(sampling_rate * seconds) if pad <= 0: return waveform arr: npt.NDArray[Any] = np.asarray(waveform) padded: npt.NDArray[Any] = np.pad(arr, (pad, 0)) return padded class ASRProcessor(ProcessorMixin): """Processor for Whisper-based ASR models.""" attributes: ClassVar[list[str]] = ["feature_extractor", "tokenizer"] feature_extractor: SequenceFeatureExtractor tokenizer: PreTrainedTokenizerBase feature_extractor_class = "AutoFeatureExtractor" tokenizer_class = "AutoTokenizer" # Fallback only. The real value comes from `ASRConfig.audio_token`, which # resolves to the decoder's native placeholder where it has one (Gemma 4's # pretrained "<|audio|>") and to "