Claude Opus 5

Reduce to the dataset-evaluation path, and stop trimming dataset clips

a7f5301 6 days ago

4.97 kB

"""Extract prosody features from recorded audio.
Audio in, the two signals the conformer consumes out: interpolated f0 and
intensity at 200 Hz. This is the front of the pipeline the paper reports --
everything after it (conformer to mel, mel to words) is unchanged whatever
the audio was.
Ported from the reference implementation's `speech.py`, which in turn ports
the training preprocessor. The settings are not interchangeable with
sensible-looking alternatives: frame_length, nccf_thresh1, the zero-padding,
the six-frame drop and the two clamps all move the contour, and the
conformer only ever saw the combination below.
"""

from __future__ import annotations

import numpy as np

from .features import FEATURE_RATE_HZ, ProsodyFeatures

# Whisper's rate, and what the preprocessor ran at.
TARGET_SAMPLE_RATE = 16000

# pYAAPT settings from the training preprocessor's own config. Every one of
# these differed from the obvious default, and the conformer only ever saw
# contours produced with these.
FRAME_LENGTH_MS = 20.0
FRAME_SPACE_MS = 5.0          # 200 Hz
TDA_FRAME_LENGTH_MS = 25.0
NCCF_THRESH1 = 0.25
# f0_min/f0_max are deliberately not passed: the preprocessor leaves yaapt's
# own defaults (60/400 Hz) in place. Passing 50/500 widened the search and
# changed which frames came back voiced.

# The preprocessor drops the first six frames, which the zero-padding below
# makes spurious.
LEADING_FRAMES_DROPPED = 6
# Clamped after interpolation, as the preprocessor does.
F0_CEILING_HZ = 500.0
INTENSITY_CLAMP_DB = 20.0

# The conformer was exported with a ten-second window, so anything past this
# is truncated rather than transcribed.
MAX_AUDIO_S = 10.0


def resample(audio: np.ndarray, orig_sr: int, target_sr: int) -> np.ndarray:
    """Linear resample, which is what the reference implementation uses."""
    audio = np.asarray(audio, dtype=np.float64)
    if orig_sr == target_sr or audio.size == 0:
        return audio
    duration = audio.size / orig_sr
    target_n = max(1, int(round(duration * target_sr)))
    source_t = np.arange(audio.size) / orig_sr
    target_t = np.arange(target_n) / target_sr
    return np.interp(target_t, source_t, audio)


def extract_features(
    audio: np.ndarray, sr: int = TARGET_SAMPLE_RATE
) -> ProsodyFeatures:
    """Audio -> the f0 and intensity the conformer consumes.
    Returns raw f0_hz and intensity_db; call `.normalised()` for the model
    input.
    The audio is used as given. Nothing here cuts a quiet lead-in or tail:
    dataset clips arrive cut to the utterance already, and re-cutting them
    on this pipeline's own intensity estimate is a step the published runs
    did not perform, so it would make the WER incomparable with theirs. A
    live microphone does need that, and the caller that has one -- the
    drawing demo -- trims before calling.
    """
    import parselmouth
    from amfm_decompy import basic_tools, pYAAPT

    from ._yaapt_patch import patched_interpolate

    audio = np.asarray(audio, dtype=np.float64)
    if audio.ndim > 1:
        audio = audio.squeeze()
    if audio.size == 0:
        return ProsodyFeatures(np.zeros(0), np.zeros(0))

    if sr != TARGET_SAMPLE_RATE:
        audio = resample(audio, sr, TARGET_SAMPLE_RATE)
        sr = TARGET_SAMPLE_RATE

    # The preprocessor patches pYAAPT's interpolation before every call and
    # reads samp_interp off the result rather than interpolating the voiced
    # values afterwards. Patch the class, not a stub, so samp_interp is
    # produced the same way.
    pYAAPT.PitchObj.interpolate = patched_interpolate

    # Zero-pad by half a frame each side before tracking. This shifts every
    # frame boundary, so without it the whole contour sits offset against
    # what training saw.
    pad = int(FRAME_LENGTH_MS / 1000 * sr) // 2
    padded = np.pad(audio, (pad, pad), "constant", constant_values=0)

    signal = basic_tools.SignalObj(padded, sr)
    pitch = pYAAPT.yaapt(
        signal,
        frame_length=FRAME_LENGTH_MS,
        frame_space=FRAME_SPACE_MS,
        nccf_thresh1=NCCF_THRESH1,
        tda_frame_length=TDA_FRAME_LENGTH_MS,
    )
    f0 = np.asarray(pitch.samp_interp, dtype=np.float64)
    f0 = f0[LEADING_FRAMES_DROPPED:]
    f0[f0 > F0_CEILING_HZ] = 0.0
    f0[f0 < 0.0] = 0.0

    # No minimum_pitch: the preprocessor passes only time_step, and
    # parselmouth's default pitch floor (100 Hz) sets the analysis window.
    # Passing 50 Hz instead doubles that window and smooths the envelope.
    sound = parselmouth.Sound(audio, sampling_frequency=sr, start_time=0)
    intensity = sound.to_intensity(time_step=1.0 / FEATURE_RATE_HZ)
    intensity_db = np.asarray(intensity.values.T.flatten(), dtype=np.float64)

    frames = min(len(f0), len(intensity_db))
    f0 = f0[:frames]
    intensity_db = intensity_db[:frames]
    intensity_db[intensity_db < INTENSITY_CLAMP_DB] = INTENSITY_CLAMP_DB

    return ProsodyFeatures(f0_hz=f0, intensity_db=intensity_db)

	"""Extract prosody features from recorded audio.

	Audio in, the two signals the conformer consumes out: interpolated f0 and
	intensity at 200 Hz. This is the front of the pipeline the paper reports --
	everything after it (conformer to mel, mel to words) is unchanged whatever
	the audio was.

	Ported from the reference implementation's `speech.py`, which in turn ports
	the training preprocessor. The settings are not interchangeable with
	sensible-looking alternatives: frame_length, nccf_thresh1, the zero-padding,
	the six-frame drop and the two clamps all move the contour, and the
	conformer only ever saw the combination below.
	"""

	from __future__ import annotations

	import numpy as np

	from .features import FEATURE_RATE_HZ, ProsodyFeatures

	# Whisper's rate, and what the preprocessor ran at.
	TARGET_SAMPLE_RATE = 16000

	# pYAAPT settings from the training preprocessor's own config. Every one of
	# these differed from the obvious default, and the conformer only ever saw
	# contours produced with these.
	FRAME_LENGTH_MS = 20.0
	FRAME_SPACE_MS = 5.0 # 200 Hz
	TDA_FRAME_LENGTH_MS = 25.0
	NCCF_THRESH1 = 0.25
	# f0_min/f0_max are deliberately not passed: the preprocessor leaves yaapt's
	# own defaults (60/400 Hz) in place. Passing 50/500 widened the search and
	# changed which frames came back voiced.

	# The preprocessor drops the first six frames, which the zero-padding below
	# makes spurious.
	LEADING_FRAMES_DROPPED = 6
	# Clamped after interpolation, as the preprocessor does.
	F0_CEILING_HZ = 500.0
	INTENSITY_CLAMP_DB = 20.0

	# The conformer was exported with a ten-second window, so anything past this
	# is truncated rather than transcribed.
	MAX_AUDIO_S = 10.0


	def resample(audio: np.ndarray, orig_sr: int, target_sr: int) -> np.ndarray:
	"""Linear resample, which is what the reference implementation uses."""
	audio = np.asarray(audio, dtype=np.float64)
	if orig_sr == target_sr or audio.size == 0:
	return audio
	duration = audio.size / orig_sr
	target_n = max(1, int(round(duration * target_sr)))
	source_t = np.arange(audio.size) / orig_sr
	target_t = np.arange(target_n) / target_sr
	return np.interp(target_t, source_t, audio)


	def extract_features(
	audio: np.ndarray, sr: int = TARGET_SAMPLE_RATE
	) -> ProsodyFeatures:
	"""Audio -> the f0 and intensity the conformer consumes.

	Returns raw f0_hz and intensity_db; call `.normalised()` for the model
	input.

	The audio is used as given. Nothing here cuts a quiet lead-in or tail:
	dataset clips arrive cut to the utterance already, and re-cutting them
	on this pipeline's own intensity estimate is a step the published runs
	did not perform, so it would make the WER incomparable with theirs. A
	live microphone does need that, and the caller that has one -- the
	drawing demo -- trims before calling.
	"""
	import parselmouth
	from amfm_decompy import basic_tools, pYAAPT

	from ._yaapt_patch import patched_interpolate

	audio = np.asarray(audio, dtype=np.float64)
	if audio.ndim > 1:
	audio = audio.squeeze()
	if audio.size == 0:
	return ProsodyFeatures(np.zeros(0), np.zeros(0))

	if sr != TARGET_SAMPLE_RATE:
	audio = resample(audio, sr, TARGET_SAMPLE_RATE)
	sr = TARGET_SAMPLE_RATE

	# The preprocessor patches pYAAPT's interpolation before every call and
	# reads samp_interp off the result rather than interpolating the voiced
	# values afterwards. Patch the class, not a stub, so samp_interp is
	# produced the same way.
	pYAAPT.PitchObj.interpolate = patched_interpolate

	# Zero-pad by half a frame each side before tracking. This shifts every
	# frame boundary, so without it the whole contour sits offset against
	# what training saw.
	pad = int(FRAME_LENGTH_MS / 1000 * sr) // 2
	padded = np.pad(audio, (pad, pad), "constant", constant_values=0)

	signal = basic_tools.SignalObj(padded, sr)
	pitch = pYAAPT.yaapt(
	signal,
	frame_length=FRAME_LENGTH_MS,
	frame_space=FRAME_SPACE_MS,
	nccf_thresh1=NCCF_THRESH1,
	tda_frame_length=TDA_FRAME_LENGTH_MS,
	)
	f0 = np.asarray(pitch.samp_interp, dtype=np.float64)
	f0 = f0[LEADING_FRAMES_DROPPED:]
	f0[f0 > F0_CEILING_HZ] = 0.0
	f0[f0 < 0.0] = 0.0

	# No minimum_pitch: the preprocessor passes only time_step, and
	# parselmouth's default pitch floor (100 Hz) sets the analysis window.
	# Passing 50 Hz instead doubles that window and smooths the envelope.
	sound = parselmouth.Sound(audio, sampling_frequency=sr, start_time=0)
	intensity = sound.to_intensity(time_step=1.0 / FEATURE_RATE_HZ)
	intensity_db = np.asarray(intensity.values.T.flatten(), dtype=np.float64)

	frames = min(len(f0), len(intensity_db))
	f0 = f0[:frames]
	intensity_db = intensity_db[:frames]
	intensity_db[intensity_db < INTENSITY_CLAMP_DB] = INTENSITY_CLAMP_DB

	return ProsodyFeatures(f0_hz=f0, intensity_db=intensity_db)