Blame for modeling_prosody.py

"""Prosody to text: f0 and intensity in, words out.

Two signals -- interpolated f0 and intensity at 200 Hz -- through a conformer
to a mel spectrogram, and the mel through a fine-tuned Whisper. No phonetic
content at any point: the model sees only how the pitch moved and how loud
it was.

Loaded the way any custom architecture on the Hub is:

    from transformers import AutoModel
    model = AutoModel.from_pretrained(
        "Daporte/prosody2text-weights", trust_remote_code=True
    )
    print(model.transcribe("utterance.wav").text)

A note on the shape of this class. It subclasses `PreTrainedModel` so that
`from_pretrained`, the `AutoModel` API and the Hub's download metrics all
work, but it is a pipeline rather than a single network: a conformer and a
`WhisperForConditionalGeneration` held alongside each other, loaded from
their own checkpoints rather than from one state dict. `forward` runs them
in sequence under `no_grad`, so this is not trainable as it stands -- it
exists to run the published pipeline, which is what the paper reports.
"""

from __future__ import annotations

from pathlib import Path

import numpy as np
import torch

from transformers import PreTrainedModel

from .configuration_prosody import ProsodyToTextConfig
from .conformer import Conformer
from .features import ProsodyFeatures
from .speech import extract_features
from .whisper_decoder import Whisper, looks_like_fallback


class ProsodyResult:
    """What one utterance produces.

    Carries the intermediate stages as well as the words, because the whole
    point is being able to see them: a transcription that looks wrong is
    usually a contour that looks wrong first.
    """

    def __init__(
        self,
        features: ProsodyFeatures,
        mel: np.ndarray,
        ranked: list[tuple[str, float]],
    ) -> None:
        self.features = features
        self.mel = mel
        self.ranked = ranked

    @property
    def text(self) -> str:
        """The chosen hypothesis, empty if nothing was transcribed."""
        return self.ranked[0][0] if self.ranked else ""

    @property
    def candidates(self) -> list[str]:
        """The whole beam. The top hypothesis is often a narrow winner on an
        input this uncertain, so the runners-up say more about what the model
        recovered than the single best string does."""
        return [text for text, _ in self.ranked]

    @property
    def fallback(self) -> bool:
        """Whether the result is Whisper's no-speech boilerplate rather than
        a transcription. Trained on captions, it answers unintelligible
        input with "Thank you." and shouted auto-caption filler, which reads
        as a result but is not one."""
        return bool(self.text) and looks_like_fallback(self.text)

    def __repr__(self) -> str:
        return (
            f"ProsodyResult(text={self.text!r}, "
            f"frames={self.features.num_frames}, mel={self.mel.shape})"
        )


class ProsodyToTextModel(PreTrainedModel):
    """The published pipeline, as a Hub model."""

    config_class = ProsodyToTextConfig
    # Nothing here is trained by this class, and the weights are loaded by
    # the two components rather than from a single state dict.
    base_model_prefix = "prosody"
    supports_gradient_checkpointing = False
    _supports_sdpa = False

    def __init__(self, config: ProsodyToTextConfig) -> None:
        super().__init__(config)
        self.config = config
        self._conformer: Conformer | None = None
        self._whisper: Whisper | None = None
        self._root: Path | None = None

    # ---- loading ---------------------------------------------------------

    @classmethod
    def from_pretrained(cls, model_id, *args, **kwargs):  # noqa: ANN001
        """Build the pipeline from a repo or a local directory.

        Overridden rather than relying on the base implementation: there is
        no single state dict to load. Each half is its own checkpoint in its
        own subfolder, so both are resolved from the snapshot and loaded by
        their own code.
        """
        from transformers import AutoConfig

        half = kwargs.pop("half", False)
        cpu = kwargs.pop("cpu", False)
        load_whisper = kwargs.pop("whisper", True)
        config = kwargs.pop("config", None)
        # Consumed so a caller can pass the usual from_pretrained arguments
        # without them reaching code that has no use for them.
        kwargs.pop("torch_dtype", None)
        kwargs.pop("device_map", None)

        root = Path(model_id)
        if not root.is_dir():
            from huggingface_hub import snapshot_download

            patterns = ["*.py", "*.json", config_file(model_id)]
            patterns += ["conformer/*", "tokenizer/*"]
            if load_whisper:
                patterns.append("whisper/*")
            root = Path(snapshot_download(
                repo_id=str(model_id),
                allow_patterns=[p for p in patterns if p],
                revision=kwargs.pop("revision", None),
                token=kwargs.pop("token", None),
            ))

        if config is None:
            config = AutoConfig.from_pretrained(
                str(root), trust_remote_code=True
            )

        model = cls(config)
        model._root = root
        model._load(root, half=half, cpu=cpu, whisper=load_whisper)
        return model

    def _load(
        self, root: Path, half: bool, cpu: bool, whisper: bool
    ) -> None:
        warn_if_driver_hidden(whisper)

        self._conformer = Conformer(
            root / self.config.conformer_subfolder, cpu=cpu
        )
        if not whisper:
            return

        tokenizer = root / self.config.tokenizer_subfolder
        self._whisper = Whisper(
            root / self.config.whisper_subfolder,
            half=half,
            num_beams=self.config.num_beams,
            tokenizer_dir=(
                tokenizer if (tokenizer / "tokenizer.json").exists() else None
            ),
        )

    @property
    def device_summary(self) -> str:
        """Where each half is running, which is worth printing: the whole
        difference between a minute and an hour is whether Whisper found the
        GPU."""
        parts = []
        if self._conformer is not None:
            parts.append(f"conformer: {self._conformer.provider}")
        if self._whisper is not None:
            parts.append(f"whisper: {self._whisper.provider}")
        return ", ".join(parts) or "nothing loaded"

    # ---- running ---------------------------------------------------------

    def forward(
        self,
        f0_hz: np.ndarray | torch.Tensor | None = None,
        intensity_db: np.ndarray | torch.Tensor | None = None,
        transcribe: bool = True,
        **kwargs,
    ) -> ProsodyResult:
        """The two contours in, the mel and the ranked beam out.

        Physical units -- Hz and dB on the 200 Hz grid -- not normalised. The
        normalisation to the model's own scale happens here so a caller
        cannot apply it twice, which produces a mel that looks plausible and
        is wrong.
        """
        if f0_hz is None or intensity_db is None:
            raise ValueError("f0_hz and intensity_db are both required")

        features = ProsodyFeatures(
            f0_hz=as_array(f0_hz), intensity_db=as_array(intensity_db)
        )
        return self.run(features, transcribe=transcribe)

    def run(
        self, features: ProsodyFeatures, transcribe: bool = True
    ) -> ProsodyResult:
        """Features to words, with the mel kept on the way through."""
        if self._conformer is None:
            raise RuntimeError(
                "the conformer is not loaded; build this with "
                "from_pretrained()"
            )
        if features.num_frames == 0:
            return ProsodyResult(
                features, np.zeros((0, self.config.mel_bins), np.float32), []
            )

        mel = self._conformer.predict(features.normalised())
        ranked: list[tuple[str, float]] = []
        if transcribe and self._whisper is not None and len(mel):
            ranked = self._whisper.hypotheses(
                mel,
                language=self.config.language,
                # None under `constrain`, so the budget comes from the
                # utterance's duration rather than the flat ceiling.
                max_tokens=(
                    None if self.config.constrain
                    else self.config.max_new_tokens
                ),
                no_repeat_ngram_size=self.config.no_repeat_ngram_size,
                constrain=self.config.constrain,
            )
        return ProsodyResult(features, mel, ranked)

    def transcribe(
        self,
        audio,
        sampling_rate: int | None = None,
        trim: bool = True,
    ) -> ProsodyResult:
        """Audio to words, for the common case.

        `audio` is a path, or samples with `sampling_rate` given. `trim` cuts
        the quiet lead-in and tail a live recording carries -- yaapt finds a
        pitch in room noise, so without it several hundred milliseconds of
        nothing reach the model. Dataset clips usually arrive trimmed
        already; pass False to match the published runs exactly.
        """
        if isinstance(audio, (str, Path)):
            # In a try block on purpose. trust_remote_code scans a modeling
            # file's imports statically and refuses to load unless every one
            # is installed -- and it skips Try nodes precisely so a module
            # can mark a dependency optional. Reading a path is the only
            # thing here that needs soundfile; a caller passing arrays, as
            # the drawing demo does, should not be made to install it.
            try:
                import soundfile
            except ImportError as error:  # pragma: no cover
                raise ImportError(
                    "reading audio from a path needs soundfile "
                    "(pip install soundfile); pass samples and "
                    "sampling_rate instead to avoid it"
                ) from error

            samples, sampling_rate = soundfile.read(
                str(audio), dtype="float64", always_2d=False
            )
            if getattr(samples, "ndim", 1) > 1:
                samples = samples.mean(axis=1)
        else:
            if sampling_rate is None:
                raise ValueError(
                    "sampling_rate is required when passing samples"
                )
            samples = as_array(audio)

        return self.run(extract_features(samples, sampling_rate, trim=trim))


# ---- helpers -------------------------------------------------------------


def as_array(value) -> np.ndarray:  # noqa: ANN001
    if isinstance(value, torch.Tensor):
        value = value.detach().cpu().numpy()
    array = np.asarray(value, dtype=np.float64)
    return array.squeeze() if array.ndim > 1 else array


def config_file(model_id) -> str:  # noqa: ANN001
    """`config.json`, named separately so the download pattern reads clearly."""
    return "config.json"


# Directories where a GPU driver may sit without being on the loader path.
#
# A pip-installed torch finds libcuda.so.1 through the dynamic loader, and on
# most systems the driver package puts it where the loader already looks. Not
# everywhere: NixOS and some container images keep it outside that path, so
# torch reports "Found no NVIDIA driver on your system" on a machine whose
# nvidia-smi works perfectly.
DRIVER_DIRS = (
    "/run/opengl-driver/lib",          # NixOS
    "/usr/lib/x86_64-linux-gnu",       # Debian, Ubuntu
    "/usr/lib64",                      # Fedora, RHEL
    "/usr/local/nvidia/lib64",         # some container images
)


def warn_if_driver_hidden(whisper: bool) -> None:
    """Say when a GPU is present but unreachable.

    Falling back to CPU silently is the wrong default for something twenty
    times slower: across a few thousand utterances that is the difference
    between an hour and a day, and the cause is one environment variable.
    """
    if not whisper:
        return
    if torch.cuda.is_available():
        return

    import ctypes

    try:
        ctypes.CDLL("libcuda.so.1")
        return          # loadable, so CUDA is unavailable for another reason
    except OSError:
        pass

    for directory in DRIVER_DIRS:
        if (Path(directory) / "libcuda.so.1").exists():
            print(
                f"warning: a GPU driver is installed but not on the library "
                f"path, so this will run on CPU. Re-run with:\n"
                f"    LD_LIBRARY_PATH={directory}:$LD_LIBRARY_PATH",
                flush=True,
            )
            return