File size: 4,965 Bytes
81cddba a7f5301 81cddba a7f5301 81cddba a7f5301 81cddba a7f5301 81cddba a7f5301 81cddba | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | """Extract prosody features from recorded audio.
Audio in, the two signals the conformer consumes out: interpolated f0 and
intensity at 200 Hz. This is the front of the pipeline the paper reports --
everything after it (conformer to mel, mel to words) is unchanged whatever
the audio was.
Ported from the reference implementation's `speech.py`, which in turn ports
the training preprocessor. The settings are not interchangeable with
sensible-looking alternatives: frame_length, nccf_thresh1, the zero-padding,
the six-frame drop and the two clamps all move the contour, and the
conformer only ever saw the combination below.
"""
from __future__ import annotations
import numpy as np
from .features import FEATURE_RATE_HZ, ProsodyFeatures
# Whisper's rate, and what the preprocessor ran at.
TARGET_SAMPLE_RATE = 16000
# pYAAPT settings from the training preprocessor's own config. Every one of
# these differed from the obvious default, and the conformer only ever saw
# contours produced with these.
FRAME_LENGTH_MS = 20.0
FRAME_SPACE_MS = 5.0 # 200 Hz
TDA_FRAME_LENGTH_MS = 25.0
NCCF_THRESH1 = 0.25
# f0_min/f0_max are deliberately not passed: the preprocessor leaves yaapt's
# own defaults (60/400 Hz) in place. Passing 50/500 widened the search and
# changed which frames came back voiced.
# The preprocessor drops the first six frames, which the zero-padding below
# makes spurious.
LEADING_FRAMES_DROPPED = 6
# Clamped after interpolation, as the preprocessor does.
F0_CEILING_HZ = 500.0
INTENSITY_CLAMP_DB = 20.0
# The conformer was exported with a ten-second window, so anything past this
# is truncated rather than transcribed.
MAX_AUDIO_S = 10.0
def resample(audio: np.ndarray, orig_sr: int, target_sr: int) -> np.ndarray:
"""Linear resample, which is what the reference implementation uses."""
audio = np.asarray(audio, dtype=np.float64)
if orig_sr == target_sr or audio.size == 0:
return audio
duration = audio.size / orig_sr
target_n = max(1, int(round(duration * target_sr)))
source_t = np.arange(audio.size) / orig_sr
target_t = np.arange(target_n) / target_sr
return np.interp(target_t, source_t, audio)
def extract_features(
audio: np.ndarray, sr: int = TARGET_SAMPLE_RATE
) -> ProsodyFeatures:
"""Audio -> the f0 and intensity the conformer consumes.
Returns raw f0_hz and intensity_db; call `.normalised()` for the model
input.
The audio is used as given. Nothing here cuts a quiet lead-in or tail:
dataset clips arrive cut to the utterance already, and re-cutting them
on this pipeline's own intensity estimate is a step the published runs
did not perform, so it would make the WER incomparable with theirs. A
live microphone does need that, and the caller that has one -- the
drawing demo -- trims before calling.
"""
import parselmouth
from amfm_decompy import basic_tools, pYAAPT
from ._yaapt_patch import patched_interpolate
audio = np.asarray(audio, dtype=np.float64)
if audio.ndim > 1:
audio = audio.squeeze()
if audio.size == 0:
return ProsodyFeatures(np.zeros(0), np.zeros(0))
if sr != TARGET_SAMPLE_RATE:
audio = resample(audio, sr, TARGET_SAMPLE_RATE)
sr = TARGET_SAMPLE_RATE
# The preprocessor patches pYAAPT's interpolation before every call and
# reads samp_interp off the result rather than interpolating the voiced
# values afterwards. Patch the class, not a stub, so samp_interp is
# produced the same way.
pYAAPT.PitchObj.interpolate = patched_interpolate
# Zero-pad by half a frame each side before tracking. This shifts every
# frame boundary, so without it the whole contour sits offset against
# what training saw.
pad = int(FRAME_LENGTH_MS / 1000 * sr) // 2
padded = np.pad(audio, (pad, pad), "constant", constant_values=0)
signal = basic_tools.SignalObj(padded, sr)
pitch = pYAAPT.yaapt(
signal,
frame_length=FRAME_LENGTH_MS,
frame_space=FRAME_SPACE_MS,
nccf_thresh1=NCCF_THRESH1,
tda_frame_length=TDA_FRAME_LENGTH_MS,
)
f0 = np.asarray(pitch.samp_interp, dtype=np.float64)
f0 = f0[LEADING_FRAMES_DROPPED:]
f0[f0 > F0_CEILING_HZ] = 0.0
f0[f0 < 0.0] = 0.0
# No minimum_pitch: the preprocessor passes only time_step, and
# parselmouth's default pitch floor (100 Hz) sets the analysis window.
# Passing 50 Hz instead doubles that window and smooths the envelope.
sound = parselmouth.Sound(audio, sampling_frequency=sr, start_time=0)
intensity = sound.to_intensity(time_step=1.0 / FEATURE_RATE_HZ)
intensity_db = np.asarray(intensity.values.T.flatten(), dtype=np.float64)
frames = min(len(f0), len(intensity_db))
f0 = f0[:frames]
intensity_db = intensity_db[:frames]
intensity_db[intensity_db < INTENSITY_CLAMP_DB] = INTENSITY_CLAMP_DB
return ProsodyFeatures(f0_hz=f0, intensity_db=intensity_db)
|