File size: 14,132 Bytes
59f824e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
155810c
 
 
 
 
59f824e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
"""Facial micro-expression feature extraction using MediaPipe Face Mesh."""

from __future__ import annotations

from typing import Dict, List, Optional, Tuple

import cv2
import numpy as np
import pandas as pd
from tqdm import tqdm

from src.utils.config import FeatureConfig, get_default_config
from src.utils.io import load_video


class FacialFeatureExtractor:
    """Extracts micro-expression features from video using MediaPipe Face Mesh."""

    def _get_mp_solutions(self):
        """Lazily load mediapipe to avoid global import errors."""
        try:
            import mediapipe as mp
            if hasattr(mp, 'solutions'):
                return mp.solutions
            # Try alternative import for some linux distributions/versions
            import mediapipe.python.solutions as solutions
            return solutions
        except Exception as e:
            raise ImportError(
                "MediaPipe 'solutions' module not found. Original error: " + str(e)
            )

    _LEFT_EYE = [33, 160, 158, 133, 153, 144]
    _RIGHT_EYE = [362, 385, 387, 263, 373, 380]
    _LEFT_BROW = [70, 63, 105, 66, 107]
    _RIGHT_BROW = [336, 296, 334, 293, 300]
    _NOSE = [1, 168, 98, 327]
    _HEAD_POINTS = [1, 33, 263, 61, 291, 199]
    _FEATURE_NAMES = [
        "eye_left_openness",
        "eye_right_openness",
        "mouth_openness",
        "mouth_width",
        "mouth_asymmetry",
        "left_brow_height",
        "right_brow_height",
        "brow_asymmetry",
        "brow_tension",
        "nose_wrinkle",
        "nostril_flare",
        "nose_asymmetry",
        "head_pitch",
        "head_yaw",
        "head_roll",
        "head_displacement",
    ]

    def __init__(self, config: Optional[FeatureConfig] = None):
        """Initialize the MediaPipe Face Mesh extractor.

        

        Gracefully handles missing MediaPipe dependencies by setting an 'available' flag.

        """

        self.config = config or get_default_config().features
        self.available = False
        self.face_mesh = None
        
        try:
            solutions = self._get_mp_solutions()
            self.mp_face_mesh = solutions.face_mesh
            self.face_mesh = self.mp_face_mesh.FaceMesh(
                static_image_mode=False,
                max_num_faces=1,
                refine_landmarks=True,
                min_detection_confidence=0.5,
                min_tracking_confidence=0.5,
            )
            self.available = True
        except (ImportError, AttributeError) as e:
            print(f"\u26a0\ufe0f WARNING: Facial features disabled. MediaPipe error: {e}")

    @staticmethod
    def _distance(point_a: np.ndarray, point_b: np.ndarray) -> float:
        """Return Euclidean distance between two landmark points."""
        return float(np.linalg.norm(point_a[:3] - point_b[:3]))

    @staticmethod
    def _face_diagonal(landmarks: np.ndarray) -> float:
        """Return face bounding-box diagonal for scale normalization."""
        if landmarks.size == 0:
            return 1.0
        mins = np.nanmin(landmarks[:, :2], axis=0)
        maxs = np.nanmax(landmarks[:, :2], axis=0)
        diagonal = float(np.linalg.norm(maxs - mins))
        return diagonal if diagonal > 1e-6 else 1.0

    @staticmethod
    def _safe_ratio(numerator: float, denominator: float) -> float:
        """Return a bounded ratio while avoiding division by zero."""
        if denominator <= 1e-8 or not np.isfinite(denominator):
            return 0.0
        value = numerator / denominator
        return float(np.clip(value, 0.0, 1.0))

    def _extract_landmarks(self, frame: np.ndarray) -> Optional[np.ndarray]:
        """Run Face Mesh on a BGR frame and return normalized landmarks."""

        if not self.available or self.face_mesh is None or frame is None or frame.size == 0:
            return None
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        if float(cv2.Laplacian(gray, cv2.CV_64F).var()) < 5.0:
            # Extremely blurry frames can cause unstable micro-expression measurements.
            return None

        rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        results = self.face_mesh.process(rgb_frame)
        if not results.multi_face_landmarks:
            return None

        height, width = frame.shape[:2]
        landmarks = np.array(
            [[point.x * width, point.y * height, point.z * width] for point in results.multi_face_landmarks[0].landmark],
            dtype=np.float32,
        )
        if self._face_diagonal(landmarks) < 25.0:
            # Very small faces do not provide reliable deception-relevant movements.
            return None
        return landmarks

    def extract_eye_openness(self, landmarks: np.ndarray) -> Tuple[float, float]:
        """Calculate left and right eye openness ratios.



        Args:

            landmarks: Face Mesh landmarks with shape ``(468, 3)``.



        Returns:

            Tuple containing left and right Eye Aspect Ratio values.

        """

        def ear(indices: List[int]) -> float:
            p1, p2, p3, p4, p5, p6 = landmarks[indices]
            vertical = self._distance(p2, p6) + self._distance(p3, p5)
            horizontal = 2.0 * self._distance(p1, p4)
            # Blink rate and eye narrowing are behavioral cues often reviewed for stress or concealment.
            return self._safe_ratio(vertical, horizontal)

        return ear(self._LEFT_EYE), ear(self._RIGHT_EYE)

    def extract_mouth_movement(self, landmarks: np.ndarray) -> Dict[str, float]:
        """Calculate scale-normalized mouth movement features.



        Args:

            landmarks: Face Mesh landmarks with shape ``(468, 3)``.



        Returns:

            Dictionary containing mouth openness, width, and asymmetry.

        """

        diagonal = self._face_diagonal(landmarks)
        upper_lip, lower_lip = landmarks[13], landmarks[14]
        left_corner, right_corner = landmarks[61], landmarks[291]
        center = (upper_lip + lower_lip) / 2.0
        left_distance = self._distance(center, left_corner)
        right_distance = self._distance(center, right_corner)
        # Mouth tension, dry-mouth behavior, and articulation changes can accompany cognitive load.
        return {
            "mouth_openness": self._distance(upper_lip, lower_lip) / diagonal,
            "mouth_width": self._distance(left_corner, right_corner) / diagonal,
            "mouth_asymmetry": abs(left_distance - right_distance) / diagonal,
        }

    def extract_eyebrow_tension(self, landmarks: np.ndarray) -> Dict[str, float]:
        """Calculate eyebrow raise, asymmetry, and squeeze features.



        Args:

            landmarks: Face Mesh landmarks with shape ``(468, 3)``.



        Returns:

            Dictionary containing brow height, asymmetry, and tension.

        """

        diagonal = self._face_diagonal(landmarks)
        nose_bridge = landmarks[168]
        left_brow = landmarks[self._LEFT_BROW]
        right_brow = landmarks[self._RIGHT_BROW]
        left_height = float(np.mean(nose_bridge[1] - left_brow[:, 1]) / diagonal)
        right_height = float(np.mean(nose_bridge[1] - right_brow[:, 1]) / diagonal)
        inner_distance = self._distance(landmarks[107], landmarks[336]) / diagonal
        # Brow squeeze and asymmetry are micro-expression proxies for stress, concern, or suppression.
        return {
            "left_brow_height": left_height,
            "right_brow_height": right_height,
            "brow_asymmetry": abs(left_height - right_height),
            "brow_tension": max(0.0, 1.0 - inner_distance),
        }

    def extract_nose_movement(self, landmarks: np.ndarray) -> Dict[str, float]:
        """Calculate nose wrinkle, nostril flare, and asymmetry features.



        Args:

            landmarks: Face Mesh landmarks with shape ``(468, 3)``.



        Returns:

            Dictionary containing nose movement measurements.

        """

        diagonal = self._face_diagonal(landmarks)
        nose_tip, nose_bridge = landmarks[1], landmarks[168]
        left_nostril, right_nostril = landmarks[98], landmarks[327]
        center = (left_nostril + right_nostril) / 2.0
        # Nose wrinkling or nostril flare can indicate disgust, arousal, or regulated emotion.
        return {
            "nose_wrinkle": abs(float(nose_tip[1] - nose_bridge[1])) / diagonal,
            "nostril_flare": self._distance(left_nostril, right_nostril) / diagonal,
            "nose_asymmetry": abs(self._distance(center, left_nostril) - self._distance(center, right_nostril)) / diagonal,
        }

    def extract_head_dynamics(

        self, landmarks: np.ndarray, prev_landmarks: Optional[np.ndarray] = None

    ) -> Dict[str, float]:
        """Calculate approximate head pose and inter-frame movement.



        Args:

            landmarks: Current Face Mesh landmarks with shape ``(468, 3)``.

            prev_landmarks: Optional previous frame landmarks for displacement.



        Returns:

            Dictionary containing pitch, yaw, roll, and displacement.

        """

        diagonal = self._face_diagonal(landmarks)
        left_eye = landmarks[33]
        right_eye = landmarks[263]
        nose_tip = landmarks[1]
        chin = landmarks[199]
        mouth_left = landmarks[61]
        mouth_right = landmarks[291]

        eye_center = (left_eye + right_eye) / 2.0
        mouth_center = (mouth_left + mouth_right) / 2.0
        eye_vector = right_eye - left_eye
        face_height = max(self._distance(eye_center, chin), 1e-6)
        face_width = max(self._distance(left_eye, right_eye), 1e-6)

        roll = float(np.degrees(np.arctan2(eye_vector[1], eye_vector[0])))
        yaw = float(((nose_tip[0] - eye_center[0]) / face_width) * 45.0)
        pitch = float(((nose_tip[1] - mouth_center[1]) / face_height) * 45.0)
        displacement = 0.0
        if prev_landmarks is not None and prev_landmarks.shape == landmarks.shape:
            # Sudden head movement or freezing can be relevant when paired with verbal hesitation.
            displacement = float(np.mean(np.linalg.norm(landmarks[self._HEAD_POINTS] - prev_landmarks[self._HEAD_POINTS], axis=1)) / diagonal)

        return {
            "head_pitch": pitch,
            "head_yaw": yaw,
            "head_roll": roll,
            "head_displacement": displacement,
        }

    def process_frame(

        self, frame: np.ndarray, prev_landmarks: Optional[np.ndarray] = None

    ) -> Optional[Dict[str, float]]:
        """Process a single BGR frame and extract all facial features.



        Args:

            frame: BGR video frame.

            prev_landmarks: Optional previous frame landmarks for dynamics.



        Returns:

            Combined feature dictionary, or ``None`` when no reliable face is detected.

        """

        landmarks = self._extract_landmarks(frame)
        if landmarks is None:
            return None

        left_eye, right_eye = self.extract_eye_openness(landmarks)
        features: Dict[str, float] = {
            "eye_left_openness": left_eye,
            "eye_right_openness": right_eye,
        }
        features.update(self.extract_mouth_movement(landmarks))
        features.update(self.extract_eyebrow_tension(landmarks))
        features.update(self.extract_nose_movement(landmarks))
        features.update(self.extract_head_dynamics(landmarks, prev_landmarks))
        features["_landmarks"] = landmarks  # Internal handoff for process_video only.
        return features

    def process_video(self, video_path: str, max_frames: Optional[int] = None) -> pd.DataFrame:
        """Process a video and extract frame-level facial features.



        Args:

            video_path: Path to a video file.

            max_frames: Optional maximum number of frames to process.



        Returns:

            DataFrame with frame index, timestamp, and feature columns. Missing

            detections are represented with ``NaN`` feature values.

        """

        capture, metadata = load_video(video_path)
        fps = float(metadata.get("fps") or 30.0)
        total_frames = int(metadata.get("frame_count") or 0)
        limit = min(total_frames, max_frames) if max_frames is not None and total_frames else max_frames or total_frames
        records: List[Dict[str, float]] = []
        prev_landmarks: Optional[np.ndarray] = None

        try:
            progress = tqdm(total=limit if limit else None, desc="Extracting facial features")
            frame_idx = 0
            while True:
                if max_frames is not None and frame_idx >= max_frames:
                    break
                ok, frame = capture.read()
                if not ok:
                    break
                record: Dict[str, float] = {"frame_idx": float(frame_idx), "timestamp": frame_idx / fps}
                features = self.process_frame(frame, prev_landmarks)
                if features is None:
                    record.update({name: np.nan for name in self.get_feature_names()})
                    prev_landmarks = None
                else:
                    landmarks = features.pop("_landmarks")
                    record.update(features)
                    prev_landmarks = landmarks
                records.append(record)
                frame_idx += 1
                progress.update(1)
            progress.close()
        finally:
            capture.release()

        columns = ["frame_idx", "timestamp"] + self.get_feature_names()
        return pd.DataFrame(records, columns=columns)

    def get_feature_names(self) -> List[str]:
        """Return ordered names of all produced facial features."""

        return list(self._FEATURE_NAMES)

    def close(self) -> None:
        """Release MediaPipe resources."""

        self.face_mesh.close()