File size: 5,759 Bytes
59f824e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
"""Visualization utilities for facial landmarks and feature timelines."""

from __future__ import annotations

from typing import Dict, Optional

import cv2
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns


class FacialVisualizer:
    """Visualizes facial landmarks and feature values on video frames."""

    def __init__(self):
        """Define landmark groups and display thresholds for overlays."""

        self.eye_landmarks = [33, 160, 158, 133, 153, 144, 362, 385, 387, 263, 373, 380]
        self.mouth_landmarks = [13, 14, 61, 291]
        self.eyebrow_landmarks = [70, 63, 105, 66, 107, 336, 296, 334, 293, 300]
        self.eye_threshold = 0.22
        self.mouth_threshold = 0.35
        self.brow_threshold = 0.18

    @staticmethod
    def _as_array(landmarks) -> np.ndarray:
        """Convert supported landmark representations to an array."""

        if landmarks is None:
            return np.empty((0, 3), dtype=np.float32)
        if isinstance(landmarks, np.ndarray):
            return landmarks
        if hasattr(landmarks, "landmark"):
            return np.array([[point.x, point.y, point.z] for point in landmarks.landmark], dtype=np.float32)
        return np.asarray(landmarks, dtype=np.float32)

    def draw_landmarks(self, frame: np.ndarray, landmarks, feature_values: Dict[str, float]) -> np.ndarray:
        """Draw face mesh landmarks and highlight active feature groups.



        Args:

            frame: BGR image frame to annotate.

            landmarks: Landmark array or MediaPipe landmark list.

            feature_values: Dictionary of extracted facial feature values.



        Returns:

            Annotated frame copy.

        """

        output = frame.copy()
        points = self._as_array(landmarks)
        if points.size == 0:
            cv2.putText(output, "No face detected", (20, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 2)
            return output

        height, width = output.shape[:2]
        if np.nanmax(points[:, :2]) <= 1.5:
            draw_points = np.column_stack((points[:, 0] * width, points[:, 1] * height)).astype(int)
        else:
            draw_points = points[:, :2].astype(int)

        for x_coord, y_coord in draw_points:
            cv2.circle(output, (int(x_coord), int(y_coord)), 1, (0, 180, 0), -1)

        eye_open = min(
            float(feature_values.get("eye_left_openness", 1.0)),
            float(feature_values.get("eye_right_openness", 1.0)),
        )
        if eye_open < self.eye_threshold:
            # Eye closure highlights possible blinking or avoidance-related stress markers.
            for idx in self.eye_landmarks:
                cv2.circle(output, tuple(draw_points[idx]), 3, (255, 0, 0), -1)

        if float(feature_values.get("mouth_openness", 0.0)) > self.mouth_threshold:
            # Mouth movement highlights speech articulation shifts and possible hesitation.
            for idx in self.mouth_landmarks:
                cv2.circle(output, tuple(draw_points[idx]), 4, (0, 0, 255), -1)

        if float(feature_values.get("brow_tension", 0.0)) > self.brow_threshold:
            # Brow tension highlights facial stress or emotion-suppression cues.
            for idx in self.eyebrow_landmarks:
                cv2.circle(output, tuple(draw_points[idx]), 3, (0, 255, 255), -1)

        overlay_items = [
            ("Eye", eye_open),
            ("Mouth", float(feature_values.get("mouth_openness", np.nan))),
            ("Brow", float(feature_values.get("brow_tension", np.nan))),
            ("Yaw", float(feature_values.get("head_yaw", np.nan))),
        ]
        for row, (name, value) in enumerate(overlay_items):
            cv2.putText(output, f"{name}: {value:.3f}", (20, 30 + row * 25), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 255), 2)
        return output

    def create_feature_timeline(self, features_df: pd.DataFrame, save_path: Optional[str] = None) -> plt.Figure:
        """Create a multi-panel timeline plot of facial features.



        Args:

            features_df: DataFrame containing frame-level facial features.

            save_path: Optional path where the figure should be saved.



        Returns:

            Matplotlib figure containing four synchronized subplots.

        """

        sns.set_theme(style="whitegrid")
        time = features_df["timestamp"] if "timestamp" in features_df else features_df.index
        fig, axes = plt.subplots(4, 1, figsize=(12, 10), sharex=True)

        axes[0].plot(time, features_df.get("eye_left_openness"), label="Left eye")
        axes[0].plot(time, features_df.get("eye_right_openness"), label="Right eye")
        axes[0].set_ylabel("Eye openness")
        axes[0].legend(loc="best")

        axes[1].plot(time, features_df.get("mouth_openness"), label="Openness")
        axes[1].plot(time, features_df.get("mouth_width"), label="Width")
        axes[1].set_ylabel("Mouth movement")
        axes[1].legend(loc="best")

        axes[2].plot(time, features_df.get("brow_tension"), label="Brow tension", color="tab:orange")
        axes[2].set_ylabel("Eyebrow tension")
        axes[2].legend(loc="best")

        for column in ["head_pitch", "head_yaw", "head_roll"]:
            axes[3].plot(time, features_df.get(column), label=column.replace("head_", "").title())
        axes[3].set_ylabel("Degrees")
        axes[3].set_xlabel("Time (s)")
        axes[3].legend(loc="best")

        fig.suptitle("Facial Feature Timeline")
        fig.tight_layout()
        if save_path:
            fig.savefig(save_path, dpi=300, bbox_inches="tight")
        return fig