File size: 2,772 Bytes
c3e6eff | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | """Configuration for the prosody-to-text pipeline."""
from __future__ import annotations
from transformers import PretrainedConfig
class ProsodyToTextConfig(PretrainedConfig):
"""Everything needed to rebuild the pipeline.
The mel contract is the part that matters. `mel_bins` and `length_ratio`
must match what the conformer was trained with: the loader falls back to
defaults when they are absent, and a wrong `length_ratio` mis-times the
mel against the audio without failing, which is the worst way for it to
be wrong.
"""
model_type = "prosody_to_text"
def __init__(
self,
# --- the conformer's mel contract -------------------------------
mel_bins: int = 12,
length_ratio: float = 0.5,
conformer_subfolder: str = "conformer",
# --- feature extraction -----------------------------------------
# 200 Hz, and the physical-range normalisation the conformer was
# trained with. These mirror the training preprocessor; changing one
# without the others silently moves the model off its own
# distribution.
feature_rate_hz: float = 200.0,
f0_scale: float = 500.0,
intensity_scale: float = 100.0,
# --- the decoder -------------------------------------------------
whisper_subfolder: str = "whisper",
tokenizer_subfolder: str = "tokenizer",
# Ten, matching the published inference runs. Wider was tried and
# made the transcriptions worse: on a signal this weak a wider beam
# searches harder for the highest-likelihood sequence, which is the
# caption boilerplate the checkpoint overlearned.
num_beams: int = 10,
language: str = "en",
no_repeat_ngram_size: int = 3,
max_new_tokens: int = 225,
# On, matching the dataset runs the published numbers come from: ban
# numerals, and take the token budget from the utterance's own
# duration rather than a flat limit. Set False to decode
# unconstrained, which is what the drawing demo does.
constrain: bool = True,
**kwargs,
) -> None:
self.mel_bins = mel_bins
self.length_ratio = length_ratio
self.conformer_subfolder = conformer_subfolder
self.feature_rate_hz = feature_rate_hz
self.f0_scale = f0_scale
self.intensity_scale = intensity_scale
self.whisper_subfolder = whisper_subfolder
self.tokenizer_subfolder = tokenizer_subfolder
self.num_beams = num_beams
self.language = language
self.no_repeat_ngram_size = no_repeat_ngram_size
self.max_new_tokens = max_new_tokens
self.constrain = constrain
super().__init__(**kwargs)
|