File size: 4,882 Bytes
59f824e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
"""Multimodal fusion utilities for facial and audio features."""

from __future__ import annotations

from typing import List, Optional

import numpy as np
import pandas as pd

from src.utils.config import AppConfig, get_default_config


class MultimodalFusion:
    """Fuses facial and audio features for multimodal deception analysis."""

    def __init__(self, config: Optional[AppConfig] = None):
        """Initialize the fusion module.



        Args:

            config: Optional application configuration.

        """

        self.config = config or get_default_config()

    def align_features(self, facial_df: pd.DataFrame, audio_df: pd.DataFrame, video_fps: float = 30.0) -> pd.DataFrame:
        """Align facial frame-level and audio segment-level features by time.



        Args:

            facial_df: Frame-level facial feature DataFrame.

            audio_df: Segment-level audio feature DataFrame with start and end times.

            video_fps: Fallback video FPS if timestamps are not present.



        Returns:

            Combined DataFrame aligned to facial frame granularity.

        """

        if facial_df.empty:
            return facial_df.copy()
        aligned = facial_df.copy().reset_index(drop=True)
        if "timestamp" not in aligned.columns:
            aligned["timestamp"] = aligned.get("frame_idx", pd.Series(aligned.index)).astype(float) / float(video_fps)
        if audio_df.empty:
            return aligned

        audio_features = [col for col in audio_df.columns if col not in {"segment_idx", "start_time", "end_time"}]
        for column in audio_features:
            aligned[column] = np.nan

        for idx, timestamp in aligned["timestamp"].items():
            matches = audio_df[(audio_df["start_time"] <= timestamp) & (timestamp < audio_df["end_time"])]
            if matches.empty and timestamp == aligned["timestamp"].iloc[-1]:
                matches = audio_df[audio_df["end_time"] <= timestamp].tail(1)
            if not matches.empty:
                aligned.loc[idx, audio_features] = matches.iloc[0][audio_features].to_numpy(dtype=float)
        return aligned.ffill().bfill()

    @staticmethod
    def _rolling_corr(left: pd.Series, right: pd.Series, window: int = 30) -> pd.Series:
        """Compute robust rolling correlation with finite fallbacks."""

        corr = left.rolling(window=window, min_periods=max(3, window // 5)).corr(right)
        return corr.replace([np.inf, -np.inf], np.nan).fillna(0.0)

    @staticmethod
    def _zscore(series: pd.Series) -> pd.Series:
        """Return z-scored values with zero variance protection."""

        std = float(series.std(skipna=True) or 0.0)
        if std <= 1e-8:
            return pd.Series(np.zeros(len(series)), index=series.index)
        return (series.fillna(series.mean()) - float(series.mean())) / std

    def extract_fusion_features(self, aligned_df: pd.DataFrame) -> pd.DataFrame:
        """Compute cross-modal fusion features.



        Args:

            aligned_df: DataFrame containing aligned facial and audio features.



        Returns:

            DataFrame with original columns plus fusion-specific features.

        """

        fused = aligned_df.copy()
        if fused.empty:
            for name in self.get_fusion_feature_names():
                fused[name] = []
            return fused

        mouth = fused.get("mouth_openness", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
        energy = fused.get("energy_mean", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
        eye = fused.get("eye_left_openness", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
        pause = fused.get("pause_count", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
        pitch = fused.get("pitch_mean", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)
        brow = fused.get("brow_tension", pd.Series(np.zeros(len(fused)), index=fused.index)).astype(float)

        # Audio-visual synchrony flags mismatches between visible articulation and speech energy.
        fused["audio_visual_sync"] = self._rolling_corr(mouth, energy)
        # Pause-gaze coupling captures whether vocal hesitation coincides with eye behavior changes.
        fused["pause_gaze_coupling"] = self._rolling_corr(eye, pause)
        # Composite stress combines vocal pitch, facial brow tension, and energy spikes.
        fused["stress_composite"] = (0.4 * self._zscore(pitch) + 0.35 * self._zscore(brow) + 0.25 * self._zscore(energy)).astype(float)
        return fused

    def get_fusion_feature_names(self) -> List[str]:
        """Return names of fusion-specific features."""

        return ["audio_visual_sync", "pause_gaze_coupling", "stress_composite"]