Spaces:
Sleeping
Sleeping
Download src/facial/extractor.py from APEXPRIT/deception-analysis-api: direct link, hf CLI and curl.
- Browser
- Download file 14.1 kB
-
https://huggingface.co/spaces/APEXPRIT/deception-analysis-api/resolve/main/src/facial/extractor.py
- Command line
-
hf download hf://spaces/APEXPRIT/deception-analysis-api/src/facial/extractor.py
-
curl -L -o extractor.py https://huggingface.co/spaces/APEXPRIT/deception-analysis-api/resolve/main/src/facial/extractor.py
14.1 kB
| """Facial micro-expression feature extraction using MediaPipe Face Mesh.""" | |
| from __future__ import annotations | |
| from typing import Dict, List, Optional, Tuple | |
| import cv2 | |
| import numpy as np | |
| import pandas as pd | |
| from tqdm import tqdm | |
| from src.utils.config import FeatureConfig, get_default_config | |
| from src.utils.io import load_video | |
| class FacialFeatureExtractor: | |
| """Extracts micro-expression features from video using MediaPipe Face Mesh.""" | |
| def _get_mp_solutions(self): | |
| """Lazily load mediapipe to avoid global import errors.""" | |
| try: | |
| import mediapipe as mp | |
| if hasattr(mp, 'solutions'): | |
| return mp.solutions | |
| # Try alternative import for some linux distributions/versions | |
| import mediapipe.python.solutions as solutions | |
| return solutions | |
| except Exception as e: | |
| raise ImportError( | |
| "MediaPipe 'solutions' module not found. Original error: " + str(e) | |
| ) | |
| _LEFT_EYE = [33, 160, 158, 133, 153, 144] | |
| _RIGHT_EYE = [362, 385, 387, 263, 373, 380] | |
| _LEFT_BROW = [70, 63, 105, 66, 107] | |
| _RIGHT_BROW = [336, 296, 334, 293, 300] | |
| _NOSE = [1, 168, 98, 327] | |
| _HEAD_POINTS = [1, 33, 263, 61, 291, 199] | |
| _FEATURE_NAMES = [ | |
| "eye_left_openness", | |
| "eye_right_openness", | |
| "mouth_openness", | |
| "mouth_width", | |
| "mouth_asymmetry", | |
| "left_brow_height", | |
| "right_brow_height", | |
| "brow_asymmetry", | |
| "brow_tension", | |
| "nose_wrinkle", | |
| "nostril_flare", | |
| "nose_asymmetry", | |
| "head_pitch", | |
| "head_yaw", | |
| "head_roll", | |
| "head_displacement", | |
| ] | |
| def __init__(self, config: Optional[FeatureConfig] = None): | |
| """Initialize the MediaPipe Face Mesh extractor. | |
| Gracefully handles missing MediaPipe dependencies by setting an 'available' flag. | |
| """ | |
| self.config = config or get_default_config().features | |
| self.available = False | |
| self.face_mesh = None | |
| try: | |
| solutions = self._get_mp_solutions() | |
| self.mp_face_mesh = solutions.face_mesh | |
| self.face_mesh = self.mp_face_mesh.FaceMesh( | |
| static_image_mode=False, | |
| max_num_faces=1, | |
| refine_landmarks=True, | |
| min_detection_confidence=0.5, | |
| min_tracking_confidence=0.5, | |
| ) | |
| self.available = True | |
| except (ImportError, AttributeError) as e: | |
| print(f"\u26a0\ufe0f WARNING: Facial features disabled. MediaPipe error: {e}") | |
| def _distance(point_a: np.ndarray, point_b: np.ndarray) -> float: | |
| """Return Euclidean distance between two landmark points.""" | |
| return float(np.linalg.norm(point_a[:3] - point_b[:3])) | |
| def _face_diagonal(landmarks: np.ndarray) -> float: | |
| """Return face bounding-box diagonal for scale normalization.""" | |
| if landmarks.size == 0: | |
| return 1.0 | |
| mins = np.nanmin(landmarks[:, :2], axis=0) | |
| maxs = np.nanmax(landmarks[:, :2], axis=0) | |
| diagonal = float(np.linalg.norm(maxs - mins)) | |
| return diagonal if diagonal > 1e-6 else 1.0 | |
| def _safe_ratio(numerator: float, denominator: float) -> float: | |
| """Return a bounded ratio while avoiding division by zero.""" | |
| if denominator <= 1e-8 or not np.isfinite(denominator): | |
| return 0.0 | |
| value = numerator / denominator | |
| return float(np.clip(value, 0.0, 1.0)) | |
| def _extract_landmarks(self, frame: np.ndarray) -> Optional[np.ndarray]: | |
| """Run Face Mesh on a BGR frame and return normalized landmarks.""" | |
| if not self.available or self.face_mesh is None or frame is None or frame.size == 0: | |
| return None | |
| gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) | |
| if float(cv2.Laplacian(gray, cv2.CV_64F).var()) < 5.0: | |
| # Extremely blurry frames can cause unstable micro-expression measurements. | |
| return None | |
| rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) | |
| results = self.face_mesh.process(rgb_frame) | |
| if not results.multi_face_landmarks: | |
| return None | |
| height, width = frame.shape[:2] | |
| landmarks = np.array( | |
| [[point.x * width, point.y * height, point.z * width] for point in results.multi_face_landmarks[0].landmark], | |
| dtype=np.float32, | |
| ) | |
| if self._face_diagonal(landmarks) < 25.0: | |
| # Very small faces do not provide reliable deception-relevant movements. | |
| return None | |
| return landmarks | |
| def extract_eye_openness(self, landmarks: np.ndarray) -> Tuple[float, float]: | |
| """Calculate left and right eye openness ratios. | |
| Args: | |
| landmarks: Face Mesh landmarks with shape ``(468, 3)``. | |
| Returns: | |
| Tuple containing left and right Eye Aspect Ratio values. | |
| """ | |
| def ear(indices: List[int]) -> float: | |
| p1, p2, p3, p4, p5, p6 = landmarks[indices] | |
| vertical = self._distance(p2, p6) + self._distance(p3, p5) | |
| horizontal = 2.0 * self._distance(p1, p4) | |
| # Blink rate and eye narrowing are behavioral cues often reviewed for stress or concealment. | |
| return self._safe_ratio(vertical, horizontal) | |
| return ear(self._LEFT_EYE), ear(self._RIGHT_EYE) | |
| def extract_mouth_movement(self, landmarks: np.ndarray) -> Dict[str, float]: | |
| """Calculate scale-normalized mouth movement features. | |
| Args: | |
| landmarks: Face Mesh landmarks with shape ``(468, 3)``. | |
| Returns: | |
| Dictionary containing mouth openness, width, and asymmetry. | |
| """ | |
| diagonal = self._face_diagonal(landmarks) | |
| upper_lip, lower_lip = landmarks[13], landmarks[14] | |
| left_corner, right_corner = landmarks[61], landmarks[291] | |
| center = (upper_lip + lower_lip) / 2.0 | |
| left_distance = self._distance(center, left_corner) | |
| right_distance = self._distance(center, right_corner) | |
| # Mouth tension, dry-mouth behavior, and articulation changes can accompany cognitive load. | |
| return { | |
| "mouth_openness": self._distance(upper_lip, lower_lip) / diagonal, | |
| "mouth_width": self._distance(left_corner, right_corner) / diagonal, | |
| "mouth_asymmetry": abs(left_distance - right_distance) / diagonal, | |
| } | |
| def extract_eyebrow_tension(self, landmarks: np.ndarray) -> Dict[str, float]: | |
| """Calculate eyebrow raise, asymmetry, and squeeze features. | |
| Args: | |
| landmarks: Face Mesh landmarks with shape ``(468, 3)``. | |
| Returns: | |
| Dictionary containing brow height, asymmetry, and tension. | |
| """ | |
| diagonal = self._face_diagonal(landmarks) | |
| nose_bridge = landmarks[168] | |
| left_brow = landmarks[self._LEFT_BROW] | |
| right_brow = landmarks[self._RIGHT_BROW] | |
| left_height = float(np.mean(nose_bridge[1] - left_brow[:, 1]) / diagonal) | |
| right_height = float(np.mean(nose_bridge[1] - right_brow[:, 1]) / diagonal) | |
| inner_distance = self._distance(landmarks[107], landmarks[336]) / diagonal | |
| # Brow squeeze and asymmetry are micro-expression proxies for stress, concern, or suppression. | |
| return { | |
| "left_brow_height": left_height, | |
| "right_brow_height": right_height, | |
| "brow_asymmetry": abs(left_height - right_height), | |
| "brow_tension": max(0.0, 1.0 - inner_distance), | |
| } | |
| def extract_nose_movement(self, landmarks: np.ndarray) -> Dict[str, float]: | |
| """Calculate nose wrinkle, nostril flare, and asymmetry features. | |
| Args: | |
| landmarks: Face Mesh landmarks with shape ``(468, 3)``. | |
| Returns: | |
| Dictionary containing nose movement measurements. | |
| """ | |
| diagonal = self._face_diagonal(landmarks) | |
| nose_tip, nose_bridge = landmarks[1], landmarks[168] | |
| left_nostril, right_nostril = landmarks[98], landmarks[327] | |
| center = (left_nostril + right_nostril) / 2.0 | |
| # Nose wrinkling or nostril flare can indicate disgust, arousal, or regulated emotion. | |
| return { | |
| "nose_wrinkle": abs(float(nose_tip[1] - nose_bridge[1])) / diagonal, | |
| "nostril_flare": self._distance(left_nostril, right_nostril) / diagonal, | |
| "nose_asymmetry": abs(self._distance(center, left_nostril) - self._distance(center, right_nostril)) / diagonal, | |
| } | |
| def extract_head_dynamics( | |
| self, landmarks: np.ndarray, prev_landmarks: Optional[np.ndarray] = None | |
| ) -> Dict[str, float]: | |
| """Calculate approximate head pose and inter-frame movement. | |
| Args: | |
| landmarks: Current Face Mesh landmarks with shape ``(468, 3)``. | |
| prev_landmarks: Optional previous frame landmarks for displacement. | |
| Returns: | |
| Dictionary containing pitch, yaw, roll, and displacement. | |
| """ | |
| diagonal = self._face_diagonal(landmarks) | |
| left_eye = landmarks[33] | |
| right_eye = landmarks[263] | |
| nose_tip = landmarks[1] | |
| chin = landmarks[199] | |
| mouth_left = landmarks[61] | |
| mouth_right = landmarks[291] | |
| eye_center = (left_eye + right_eye) / 2.0 | |
| mouth_center = (mouth_left + mouth_right) / 2.0 | |
| eye_vector = right_eye - left_eye | |
| face_height = max(self._distance(eye_center, chin), 1e-6) | |
| face_width = max(self._distance(left_eye, right_eye), 1e-6) | |
| roll = float(np.degrees(np.arctan2(eye_vector[1], eye_vector[0]))) | |
| yaw = float(((nose_tip[0] - eye_center[0]) / face_width) * 45.0) | |
| pitch = float(((nose_tip[1] - mouth_center[1]) / face_height) * 45.0) | |
| displacement = 0.0 | |
| if prev_landmarks is not None and prev_landmarks.shape == landmarks.shape: | |
| # Sudden head movement or freezing can be relevant when paired with verbal hesitation. | |
| displacement = float(np.mean(np.linalg.norm(landmarks[self._HEAD_POINTS] - prev_landmarks[self._HEAD_POINTS], axis=1)) / diagonal) | |
| return { | |
| "head_pitch": pitch, | |
| "head_yaw": yaw, | |
| "head_roll": roll, | |
| "head_displacement": displacement, | |
| } | |
| def process_frame( | |
| self, frame: np.ndarray, prev_landmarks: Optional[np.ndarray] = None | |
| ) -> Optional[Dict[str, float]]: | |
| """Process a single BGR frame and extract all facial features. | |
| Args: | |
| frame: BGR video frame. | |
| prev_landmarks: Optional previous frame landmarks for dynamics. | |
| Returns: | |
| Combined feature dictionary, or ``None`` when no reliable face is detected. | |
| """ | |
| landmarks = self._extract_landmarks(frame) | |
| if landmarks is None: | |
| return None | |
| left_eye, right_eye = self.extract_eye_openness(landmarks) | |
| features: Dict[str, float] = { | |
| "eye_left_openness": left_eye, | |
| "eye_right_openness": right_eye, | |
| } | |
| features.update(self.extract_mouth_movement(landmarks)) | |
| features.update(self.extract_eyebrow_tension(landmarks)) | |
| features.update(self.extract_nose_movement(landmarks)) | |
| features.update(self.extract_head_dynamics(landmarks, prev_landmarks)) | |
| features["_landmarks"] = landmarks # Internal handoff for process_video only. | |
| return features | |
| def process_video(self, video_path: str, max_frames: Optional[int] = None) -> pd.DataFrame: | |
| """Process a video and extract frame-level facial features. | |
| Args: | |
| video_path: Path to a video file. | |
| max_frames: Optional maximum number of frames to process. | |
| Returns: | |
| DataFrame with frame index, timestamp, and feature columns. Missing | |
| detections are represented with ``NaN`` feature values. | |
| """ | |
| capture, metadata = load_video(video_path) | |
| fps = float(metadata.get("fps") or 30.0) | |
| total_frames = int(metadata.get("frame_count") or 0) | |
| limit = min(total_frames, max_frames) if max_frames is not None and total_frames else max_frames or total_frames | |
| records: List[Dict[str, float]] = [] | |
| prev_landmarks: Optional[np.ndarray] = None | |
| try: | |
| progress = tqdm(total=limit if limit else None, desc="Extracting facial features") | |
| frame_idx = 0 | |
| while True: | |
| if max_frames is not None and frame_idx >= max_frames: | |
| break | |
| ok, frame = capture.read() | |
| if not ok: | |
| break | |
| record: Dict[str, float] = {"frame_idx": float(frame_idx), "timestamp": frame_idx / fps} | |
| features = self.process_frame(frame, prev_landmarks) | |
| if features is None: | |
| record.update({name: np.nan for name in self.get_feature_names()}) | |
| prev_landmarks = None | |
| else: | |
| landmarks = features.pop("_landmarks") | |
| record.update(features) | |
| prev_landmarks = landmarks | |
| records.append(record) | |
| frame_idx += 1 | |
| progress.update(1) | |
| progress.close() | |
| finally: | |
| capture.release() | |
| columns = ["frame_idx", "timestamp"] + self.get_feature_names() | |
| return pd.DataFrame(records, columns=columns) | |
| def get_feature_names(self) -> List[str]: | |
| """Return ordered names of all produced facial features.""" | |
| return list(self._FEATURE_NAMES) | |
| def close(self) -> None: | |
| """Release MediaPipe resources.""" | |
| self.face_mesh.close() | |