File size: 11,108 Bytes
59f824e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
"""Audio and speech feature extraction using librosa."""

from __future__ import annotations

from typing import Dict, List, Optional

import librosa
import numpy as np
import pandas as pd

from src.utils.config import AudioConfig, get_default_config


class AudioFeatureExtractor:
    """Extracts speech and audio features for deception analysis using librosa."""

    def __init__(self, config: Optional[AudioConfig] = None):
        """Initialize the audio extractor.



        Args:

            config: Optional audio configuration. Defaults to application audio config.

        """

        self.config = config or get_default_config().audio

    @staticmethod
    def _finite_or_zero(value: float) -> float:
        """Convert non-finite numeric outputs to zero."""

        return float(value) if np.isfinite(value) else 0.0

    def _empty_features(self) -> Dict[str, float]:
        """Return a zero-filled feature dictionary for invalid audio."""

        return {name: 0.0 for name in self.get_feature_names()}

    def extract_pitch_variation(self, y: np.ndarray, sr: int) -> Dict[str, float]:
        """Extract pitch/f0 features using probabilistic YIN.



        Args:

            y: Audio waveform.

            sr: Sample rate.



        Returns:

            Dictionary with pitch summary and voiced-frame ratio.

        """

        if y.size < self.config.hop_length or np.allclose(y, 0.0):
            return {"pitch_mean": 0.0, "pitch_std": 0.0, "pitch_range": 0.0, "pitch_variation_coefficient": 0.0, "voiced_ratio": 0.0}
        try:
            f0, _, voiced_prob = librosa.pyin(y, fmin=65.0, fmax=2093.0, sr=sr, hop_length=self.config.hop_length)
        except Exception:
            return {"pitch_mean": 0.0, "pitch_std": 0.0, "pitch_range": 0.0, "pitch_variation_coefficient": 0.0, "voiced_ratio": 0.0}
        voiced = f0[np.isfinite(f0)]
        if voiced.size == 0:
            return {"pitch_mean": 0.0, "pitch_std": 0.0, "pitch_range": 0.0, "pitch_variation_coefficient": 0.0, "voiced_ratio": 0.0}
        mean = float(np.mean(voiced))
        std = float(np.std(voiced))
        # Pitch elevation and instability can reflect stress or high cognitive load during deceptive speech.
        return {
            "pitch_mean": mean,
            "pitch_std": std,
            "pitch_range": float(np.max(voiced) - np.min(voiced)),
            "pitch_variation_coefficient": self._finite_or_zero(std / mean if mean else 0.0),
            "voiced_ratio": float(np.mean(np.asarray(voiced_prob) > 0.5)) if voiced_prob is not None else float(voiced.size / f0.size),
        }

    def extract_pause_analysis(self, y: np.ndarray, sr: int) -> Dict[str, float]:
        """Analyze speech pauses and timing with adaptive energy VAD.



        Args:

            y: Audio waveform.

            sr: Sample rate.



        Returns:

            Dictionary with pause count, duration, rate, and speech ratio.

        """

        duration = len(y) / float(sr) if sr else 0.0
        if y.size == 0 or duration <= 0.0:
            return {"pause_count": 0.0, "avg_pause_duration": 0.0, "max_pause_duration": 0.0, "pause_rate": 0.0, "speech_ratio": 0.0}
        rms = librosa.feature.rms(y=y, frame_length=self.config.n_fft, hop_length=self.config.hop_length)[0]
        if rms.size == 0 or np.allclose(rms, 0.0):
            return {"pause_count": 1.0, "avg_pause_duration": duration, "max_pause_duration": duration, "pause_rate": 60.0 / duration, "speech_ratio": 0.0}
        threshold = max(float(np.median(rms) * 0.6), float(np.max(rms) * 0.03))
        silent = rms <= threshold
        frame_seconds = self.config.hop_length / float(sr)
        min_pause_frames = max(1, int(np.ceil(0.3 / frame_seconds)))
        pauses: List[float] = []
        run = 0
        for is_silent in silent:
            if is_silent:
                run += 1
            else:
                if run >= min_pause_frames:
                    pauses.append(run * frame_seconds)
                run = 0
        if run >= min_pause_frames:
            pauses.append(run * frame_seconds)
        pause_time = float(np.sum(pauses)) if pauses else 0.0
        # Pauses and hesitations are deception-relevant when they indicate formulation difficulty.
        return {
            "pause_count": float(len(pauses)),
            "avg_pause_duration": float(np.mean(pauses)) if pauses else 0.0,
            "max_pause_duration": float(np.max(pauses)) if pauses else 0.0,
            "pause_rate": float(len(pauses) / (duration / 60.0)) if duration > 0 else 0.0,
            "speech_ratio": float(np.clip(1.0 - pause_time / duration, 0.0, 1.0)),
        }

    def extract_speech_energy(self, y: np.ndarray, sr: int) -> Dict[str, float]:
        """Extract speech energy and spectral intensity features.



        Args:

            y: Audio waveform.

            sr: Sample rate.



        Returns:

            Dictionary with RMS energy and spectral descriptors.

        """

        if y.size == 0:
            return {"energy_mean": 0.0, "energy_std": 0.0, "energy_range": 0.0, "energy_variation": 0.0, "spectral_centroid_mean": 0.0, "spectral_bandwidth_mean": 0.0}
        rms = librosa.feature.rms(y=y, frame_length=self.config.n_fft, hop_length=self.config.hop_length)[0]
        mean = float(np.mean(rms)) if rms.size else 0.0
        std = float(np.std(rms)) if rms.size else 0.0
        centroid = librosa.feature.spectral_centroid(y=y, sr=sr, n_fft=self.config.n_fft, hop_length=self.config.hop_length)[0]
        bandwidth = librosa.feature.spectral_bandwidth(y=y, sr=sr, n_fft=self.config.n_fft, hop_length=self.config.hop_length)[0]
        # Vocal intensity spikes can pair with facial stress markers in multimodal deception cues.
        return {
            "energy_mean": mean,
            "energy_std": std,
            "energy_range": float(np.max(rms) - np.min(rms)) if rms.size else 0.0,
            "energy_variation": self._finite_or_zero(std / mean if mean else 0.0),
            "spectral_centroid_mean": float(np.mean(centroid)) if centroid.size else 0.0,
            "spectral_bandwidth_mean": float(np.mean(bandwidth)) if bandwidth.size else 0.0,
        }

    def extract_mfcc_features(self, y: np.ndarray, sr: int) -> Dict[str, float]:
        """Extract MFCC and delta-MFCC speech characterization features.



        Args:

            y: Audio waveform.

            sr: Sample rate.



        Returns:

            Dictionary containing MFCC means, standard deviations, and delta means.

        """

        features: Dict[str, float] = {}
        if y.size == 0:
            for idx in range(self.config.n_mfcc):
                features[f"mfcc_{idx}_mean"] = 0.0
                features[f"mfcc_{idx}_std"] = 0.0
                features[f"delta_mfcc_{idx}_mean"] = 0.0
            return features
        mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=self.config.n_mfcc, n_fft=self.config.n_fft, hop_length=self.config.hop_length)
        delta = librosa.feature.delta(mfcc) if mfcc.shape[1] >= 3 else np.zeros_like(mfcc)
        # MFCCs capture voice quality shifts that may accompany stress or controlled speech.
        for idx in range(self.config.n_mfcc):
            features[f"mfcc_{idx}_mean"] = float(np.mean(mfcc[idx]))
            features[f"mfcc_{idx}_std"] = float(np.std(mfcc[idx]))
            features[f"delta_mfcc_{idx}_mean"] = float(np.mean(delta[idx]))
        return features

    def process_audio(self, audio_path: str) -> Dict[str, float]:
        """Process a complete audio file and extract all features.



        Args:

            audio_path: Path to an audio file readable by librosa.



        Returns:

            Combined audio feature dictionary.

        """

        try:
            y, sr = librosa.load(audio_path, sr=self.config.sample_rate, mono=True)
        except Exception:
            return self._empty_features()
        if y.size == 0:
            return self._empty_features()
        features: Dict[str, float] = {}
        features.update(self.extract_pitch_variation(y, sr))
        features.update(self.extract_pause_analysis(y, sr))
        features.update(self.extract_speech_energy(y, sr))
        features.update(self.extract_mfcc_features(y, sr))
        return {name: self._finite_or_zero(float(features.get(name, 0.0))) for name in self.get_feature_names()}

    def process_audio_segments(self, audio_path: str, segment_duration: float = 2.0) -> pd.DataFrame:
        """Process audio in fixed-duration segments for temporal analysis.



        Args:

            audio_path: Path to an audio file readable by librosa.

            segment_duration: Segment length in seconds.



        Returns:

            DataFrame with segment index, timing, and audio feature columns.

        """

        try:
            y, sr = librosa.load(audio_path, sr=self.config.sample_rate, mono=True)
        except Exception:
            columns = ["segment_idx", "start_time", "end_time"] + self.get_feature_names()
            return pd.DataFrame(columns=columns)
        segment_samples = max(1, int(segment_duration * sr))
        records: List[Dict[str, float]] = []
        for segment_idx, start in enumerate(range(0, len(y), segment_samples)):
            end = min(start + segment_samples, len(y))
            segment = y[start:end]
            record: Dict[str, float] = {
                "segment_idx": float(segment_idx),
                "start_time": start / float(sr),
                "end_time": end / float(sr),
            }
            if segment.size < max(16, self.config.hop_length // 2):
                record.update(self._empty_features())
            else:
                record.update(self.extract_pitch_variation(segment, sr))
                record.update(self.extract_pause_analysis(segment, sr))
                record.update(self.extract_speech_energy(segment, sr))
                record.update(self.extract_mfcc_features(segment, sr))
            records.append(record)
        return pd.DataFrame(records, columns=["segment_idx", "start_time", "end_time"] + self.get_feature_names())

    def get_feature_names(self) -> List[str]:
        """Return ordered names of all produced audio features."""

        names = [
            "pitch_mean",
            "pitch_std",
            "pitch_range",
            "pitch_variation_coefficient",
            "voiced_ratio",
            "pause_count",
            "avg_pause_duration",
            "max_pause_duration",
            "pause_rate",
            "speech_ratio",
            "energy_mean",
            "energy_std",
            "energy_range",
            "energy_variation",
            "spectral_centroid_mean",
            "spectral_bandwidth_mean",
        ]
        for idx in range(self.config.n_mfcc):
            names.extend([f"mfcc_{idx}_mean", f"mfcc_{idx}_std", f"delta_mfcc_{idx}_mean"])
        return names