deception-analysis-api / src /audio /extractor.py
APEXPRIT's picture
Upload folder using huggingface_hub
59f824e verified
Raw History Blame Contribute Delete
11.1 kB
"""Audio and speech feature extraction using librosa."""
from __future__ import annotations
from typing import Dict, List, Optional
import librosa
import numpy as np
import pandas as pd
from src.utils.config import AudioConfig, get_default_config
class AudioFeatureExtractor:
"""Extracts speech and audio features for deception analysis using librosa."""
def __init__(self, config: Optional[AudioConfig] = None):
"""Initialize the audio extractor.
Args:
config: Optional audio configuration. Defaults to application audio config.
"""
self.config = config or get_default_config().audio
@staticmethod
def _finite_or_zero(value: float) -> float:
"""Convert non-finite numeric outputs to zero."""
return float(value) if np.isfinite(value) else 0.0
def _empty_features(self) -> Dict[str, float]:
"""Return a zero-filled feature dictionary for invalid audio."""
return {name: 0.0 for name in self.get_feature_names()}
def extract_pitch_variation(self, y: np.ndarray, sr: int) -> Dict[str, float]:
"""Extract pitch/f0 features using probabilistic YIN.
Args:
y: Audio waveform.
sr: Sample rate.
Returns:
Dictionary with pitch summary and voiced-frame ratio.
"""
if y.size < self.config.hop_length or np.allclose(y, 0.0):
return {"pitch_mean": 0.0, "pitch_std": 0.0, "pitch_range": 0.0, "pitch_variation_coefficient": 0.0, "voiced_ratio": 0.0}
try:
f0, _, voiced_prob = librosa.pyin(y, fmin=65.0, fmax=2093.0, sr=sr, hop_length=self.config.hop_length)
except Exception:
return {"pitch_mean": 0.0, "pitch_std": 0.0, "pitch_range": 0.0, "pitch_variation_coefficient": 0.0, "voiced_ratio": 0.0}
voiced = f0[np.isfinite(f0)]
if voiced.size == 0:
return {"pitch_mean": 0.0, "pitch_std": 0.0, "pitch_range": 0.0, "pitch_variation_coefficient": 0.0, "voiced_ratio": 0.0}
mean = float(np.mean(voiced))
std = float(np.std(voiced))
# Pitch elevation and instability can reflect stress or high cognitive load during deceptive speech.
return {
"pitch_mean": mean,
"pitch_std": std,
"pitch_range": float(np.max(voiced) - np.min(voiced)),
"pitch_variation_coefficient": self._finite_or_zero(std / mean if mean else 0.0),
"voiced_ratio": float(np.mean(np.asarray(voiced_prob) > 0.5)) if voiced_prob is not None else float(voiced.size / f0.size),
}
def extract_pause_analysis(self, y: np.ndarray, sr: int) -> Dict[str, float]:
"""Analyze speech pauses and timing with adaptive energy VAD.
Args:
y: Audio waveform.
sr: Sample rate.
Returns:
Dictionary with pause count, duration, rate, and speech ratio.
"""
duration = len(y) / float(sr) if sr else 0.0
if y.size == 0 or duration <= 0.0:
return {"pause_count": 0.0, "avg_pause_duration": 0.0, "max_pause_duration": 0.0, "pause_rate": 0.0, "speech_ratio": 0.0}
rms = librosa.feature.rms(y=y, frame_length=self.config.n_fft, hop_length=self.config.hop_length)[0]
if rms.size == 0 or np.allclose(rms, 0.0):
return {"pause_count": 1.0, "avg_pause_duration": duration, "max_pause_duration": duration, "pause_rate": 60.0 / duration, "speech_ratio": 0.0}
threshold = max(float(np.median(rms) * 0.6), float(np.max(rms) * 0.03))
silent = rms <= threshold
frame_seconds = self.config.hop_length / float(sr)
min_pause_frames = max(1, int(np.ceil(0.3 / frame_seconds)))
pauses: List[float] = []
run = 0
for is_silent in silent:
if is_silent:
run += 1
else:
if run >= min_pause_frames:
pauses.append(run * frame_seconds)
run = 0
if run >= min_pause_frames:
pauses.append(run * frame_seconds)
pause_time = float(np.sum(pauses)) if pauses else 0.0
# Pauses and hesitations are deception-relevant when they indicate formulation difficulty.
return {
"pause_count": float(len(pauses)),
"avg_pause_duration": float(np.mean(pauses)) if pauses else 0.0,
"max_pause_duration": float(np.max(pauses)) if pauses else 0.0,
"pause_rate": float(len(pauses) / (duration / 60.0)) if duration > 0 else 0.0,
"speech_ratio": float(np.clip(1.0 - pause_time / duration, 0.0, 1.0)),
}
def extract_speech_energy(self, y: np.ndarray, sr: int) -> Dict[str, float]:
"""Extract speech energy and spectral intensity features.
Args:
y: Audio waveform.
sr: Sample rate.
Returns:
Dictionary with RMS energy and spectral descriptors.
"""
if y.size == 0:
return {"energy_mean": 0.0, "energy_std": 0.0, "energy_range": 0.0, "energy_variation": 0.0, "spectral_centroid_mean": 0.0, "spectral_bandwidth_mean": 0.0}
rms = librosa.feature.rms(y=y, frame_length=self.config.n_fft, hop_length=self.config.hop_length)[0]
mean = float(np.mean(rms)) if rms.size else 0.0
std = float(np.std(rms)) if rms.size else 0.0
centroid = librosa.feature.spectral_centroid(y=y, sr=sr, n_fft=self.config.n_fft, hop_length=self.config.hop_length)[0]
bandwidth = librosa.feature.spectral_bandwidth(y=y, sr=sr, n_fft=self.config.n_fft, hop_length=self.config.hop_length)[0]
# Vocal intensity spikes can pair with facial stress markers in multimodal deception cues.
return {
"energy_mean": mean,
"energy_std": std,
"energy_range": float(np.max(rms) - np.min(rms)) if rms.size else 0.0,
"energy_variation": self._finite_or_zero(std / mean if mean else 0.0),
"spectral_centroid_mean": float(np.mean(centroid)) if centroid.size else 0.0,
"spectral_bandwidth_mean": float(np.mean(bandwidth)) if bandwidth.size else 0.0,
}
def extract_mfcc_features(self, y: np.ndarray, sr: int) -> Dict[str, float]:
"""Extract MFCC and delta-MFCC speech characterization features.
Args:
y: Audio waveform.
sr: Sample rate.
Returns:
Dictionary containing MFCC means, standard deviations, and delta means.
"""
features: Dict[str, float] = {}
if y.size == 0:
for idx in range(self.config.n_mfcc):
features[f"mfcc_{idx}_mean"] = 0.0
features[f"mfcc_{idx}_std"] = 0.0
features[f"delta_mfcc_{idx}_mean"] = 0.0
return features
mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=self.config.n_mfcc, n_fft=self.config.n_fft, hop_length=self.config.hop_length)
delta = librosa.feature.delta(mfcc) if mfcc.shape[1] >= 3 else np.zeros_like(mfcc)
# MFCCs capture voice quality shifts that may accompany stress or controlled speech.
for idx in range(self.config.n_mfcc):
features[f"mfcc_{idx}_mean"] = float(np.mean(mfcc[idx]))
features[f"mfcc_{idx}_std"] = float(np.std(mfcc[idx]))
features[f"delta_mfcc_{idx}_mean"] = float(np.mean(delta[idx]))
return features
def process_audio(self, audio_path: str) -> Dict[str, float]:
"""Process a complete audio file and extract all features.
Args:
audio_path: Path to an audio file readable by librosa.
Returns:
Combined audio feature dictionary.
"""
try:
y, sr = librosa.load(audio_path, sr=self.config.sample_rate, mono=True)
except Exception:
return self._empty_features()
if y.size == 0:
return self._empty_features()
features: Dict[str, float] = {}
features.update(self.extract_pitch_variation(y, sr))
features.update(self.extract_pause_analysis(y, sr))
features.update(self.extract_speech_energy(y, sr))
features.update(self.extract_mfcc_features(y, sr))
return {name: self._finite_or_zero(float(features.get(name, 0.0))) for name in self.get_feature_names()}
def process_audio_segments(self, audio_path: str, segment_duration: float = 2.0) -> pd.DataFrame:
"""Process audio in fixed-duration segments for temporal analysis.
Args:
audio_path: Path to an audio file readable by librosa.
segment_duration: Segment length in seconds.
Returns:
DataFrame with segment index, timing, and audio feature columns.
"""
try:
y, sr = librosa.load(audio_path, sr=self.config.sample_rate, mono=True)
except Exception:
columns = ["segment_idx", "start_time", "end_time"] + self.get_feature_names()
return pd.DataFrame(columns=columns)
segment_samples = max(1, int(segment_duration * sr))
records: List[Dict[str, float]] = []
for segment_idx, start in enumerate(range(0, len(y), segment_samples)):
end = min(start + segment_samples, len(y))
segment = y[start:end]
record: Dict[str, float] = {
"segment_idx": float(segment_idx),
"start_time": start / float(sr),
"end_time": end / float(sr),
}
if segment.size < max(16, self.config.hop_length // 2):
record.update(self._empty_features())
else:
record.update(self.extract_pitch_variation(segment, sr))
record.update(self.extract_pause_analysis(segment, sr))
record.update(self.extract_speech_energy(segment, sr))
record.update(self.extract_mfcc_features(segment, sr))
records.append(record)
return pd.DataFrame(records, columns=["segment_idx", "start_time", "end_time"] + self.get_feature_names())
def get_feature_names(self) -> List[str]:
"""Return ordered names of all produced audio features."""
names = [
"pitch_mean",
"pitch_std",
"pitch_range",
"pitch_variation_coefficient",
"voiced_ratio",
"pause_count",
"avg_pause_duration",
"max_pause_duration",
"pause_rate",
"speech_ratio",
"energy_mean",
"energy_std",
"energy_range",
"energy_variation",
"spectral_centroid_mean",
"spectral_bandwidth_mean",
]
for idx in range(self.config.n_mfcc):
names.extend([f"mfcc_{idx}_mean", f"mfcc_{idx}_std", f"delta_mfcc_{idx}_mean"])
return names