File size: 14,751 Bytes
8d7b4b1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
"""Modular runtime feature extractors for live inference.

These extractors automatically generate training-compatible features from raw
webcam/video/audio input without manual configuration.
"""

from __future__ import annotations

import csv
from pathlib import Path
from typing import Dict, List, Optional, Sequence
import numpy as np
import torch
import librosa
import torchvision.models as models
import torchvision.transforms as T
from transformers import Wav2Vec2FeatureExtractor, Wav2Vec2Model
from sentence_transformers import SentenceTransformer

from src.features.sanitization import sanitize_array
from src.features.preprocessing import FeaturePreprocessor, PCATransform, FeatureNormalizer
from src.features.specs import AudioSpec, VideoSpec, TextSpec


class OpenFaceRuntimeExtractor:
    """Extract OpenFace behavioral features from video frames or CSV.

    Supports:
    - Direct frame-based extraction (requires OpenFace installed)
    - CSV loading for pre-computed OpenFace features
    """

    EXPECTED_COLUMNS = [
        "AU01_r", "AU02_r", "AU04_r", "AU05_r", "AU06_r", "AU07_r", "AU09_r",
        "AU10_r", "AU12_r", "AU14_r", "AU15_r", "AU17_r", "AU20_r", "AU23_r",
        "AU25_r", "AU26_r", "AU45_r",  # 17 AUs
        "Gaze_angle_x", "Gaze_angle_y", "Gaze_angle_z",  # 3 gaze angles
        "Pose_Tx", "Pose_Ty", "Pose_Tz",  # 3 head pose
    ]

    def __init__(self, use_csv: bool = True):
        """Initialize extractor.

        Args:
            use_csv: If True, load from CSV. If False, use frame-based extraction.
        """
        self.use_csv = use_csv

    def extract_from_csv(self, csv_path: Path) -> np.ndarray:
        """Extract OpenFace features from CSV file.

        Args:
            csv_path: Path to OpenFace CSV file

        Returns:
            Features array [num_frames, num_features]
        """
        if not csv_path.exists():
            raise FileNotFoundError(f"OpenFace CSV not found: {csv_path}")

        with open(csv_path, "r", encoding="utf-8") as f:
            reader = csv.DictReader(f)
            rows = list(reader)

        if not rows:
            raise ValueError(f"OpenFace CSV is empty: {csv_path}")

        # Extract features
        features_list = []
        for row in rows:
            try:
                # Extract AU features
                au_features = [float(row.get(col, 0.0)) for col in self.EXPECTED_COLUMNS[:17]]
                # Extract gaze
                gaze_features = [float(row.get(col, 0.0)) for col in self.EXPECTED_COLUMNS[17:20]]
                # Extract head pose
                pose_features = [float(row.get(col, 0.0)) for col in self.EXPECTED_COLUMNS[20:23]]
                
                features_list.append(au_features + gaze_features + pose_features)
            except (ValueError, KeyError) as e:
                continue

        if not features_list:
            raise ValueError(f"Could not extract valid features from {csv_path}")

        features = np.array(features_list, dtype=np.float32)
        return sanitize_array(features)

    def extract(self, source: Path | str) -> np.ndarray:
        """Extract OpenFace features from CSV or video frames."""
        source_path = Path(source) if isinstance(source, str) else source
        
        if source_path.suffix == ".csv":
            return self.extract_from_csv(source_path)
        else:
            raise NotImplementedError(
                "Frame-based OpenFace extraction requires additional dependencies. "
                "Use CSV format for now."
            )


class ResNetRuntimeExtractor:
    """Extract visual embeddings from video frames using ResNet50.

    Supports:
    - Frame-by-frame extraction
    - Temporal aggregation
    - PCA reduction for consistency with training
    """

    def __init__(
        self,
        device: str = "auto",
        pca_preprocessor: Optional[FeaturePreprocessor] = None,
    ):
        """Initialize extractor.

        Args:
            device: Device to use ("auto", "cuda", "cpu")
            pca_preprocessor: Optional PCA preprocessor for dimensionality reduction
        """
        if device == "auto":
            device = "cuda" if torch.cuda.is_available() else "cpu"
        
        self.device = torch.device(device)
        self.pca_preprocessor = pca_preprocessor

        # Load pretrained ResNet50
        model = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V1)
        self.backbone = torch.nn.Sequential(*list(model.children())[:-1]).to(self.device)
        self.backbone.eval()

        # Image preprocessing
        self.transform = T.Compose([
            T.Resize((224, 224)),
            T.ToTensor(),
            T.Normalize(
                mean=[0.485, 0.456, 0.406],
                std=[0.229, 0.224, 0.225],
            ),
        ])

    def extract_from_frames(self, frames: Sequence[np.ndarray]) -> np.ndarray:
        """Extract embeddings from video frames.

        Args:
            frames: List of frames as np.ndarray [H, W, 3]

        Returns:
            Embeddings array [num_frames, embedding_dim]
        """
        if not frames:
            raise ValueError("No frames provided")

        embeddings_list = []
        with torch.no_grad():
            for frame in frames:
                if isinstance(frame, np.ndarray):
                    # Convert BGR to RGB if needed
                    if frame.shape[2] == 3:
                        if frame.max() > 1.0:  # Likely 0-255 range
                            frame = frame.astype(np.float32) / 255.0
                        frame_tensor = torch.from_numpy(frame).permute(2, 0, 1).float()
                    else:
                        frame_tensor = self.transform(T.ToPILImage()(frame))
                else:
                    frame_tensor = self.transform(frame)

                frame_tensor = frame_tensor.unsqueeze(0).to(self.device)
                embedding = self.backbone(frame_tensor).squeeze()
                embeddings_list.append(embedding.cpu().numpy())

        embeddings = np.array(embeddings_list, dtype=np.float32)  # [num_frames, 2048]
        embeddings = sanitize_array(embeddings)

        # Apply PCA if available
        if self.pca_preprocessor:
            embeddings = self.pca_preprocessor.transform(embeddings)

        return embeddings


class AudioFeatureRuntimeExtractor:
    """Extract audio features: MFCC + HuBERT.

    Produces training-compatible audio features.
    """

    def __init__(
        self,
        sample_rate: int = 16000,
        mfcc_n_mfcc: int = 13,
        use_hubert: bool = False,
        normalizer: Optional[FeatureNormalizer] = None,
    ):
        """Initialize extractor.

        Args:
            sample_rate: Audio sample rate
            mfcc_n_mfcc: Number of MFCC coefficients
            use_hubert: Whether to include HuBERT embeddings
            normalizer: Optional normalizer for features
        """
        self.sample_rate = sample_rate
        self.mfcc_n_mfcc = mfcc_n_mfcc
        self.use_hubert = use_hubert
        self.normalizer = normalizer

        if use_hubert:
            self._hubert_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
                "facebook/hubert-base-ls960"
            )
            self._hubert_model = Wav2Vec2Model.from_pretrained("facebook/hubert-base-ls960")
            self._hubert_model.eval()
            if torch.cuda.is_available():
                self._hubert_model = self._hubert_model.cuda()

    def extract_mfcc(self, waveform: np.ndarray) -> np.ndarray:
        """Extract MFCC features: 13 MFCC + 13 delta + 13 delta-delta.

        Args:
            waveform: Audio waveform [num_samples]

        Returns:
            MFCC features [num_frames, 39]
        """
        waveform = np.asarray(waveform, dtype=np.float32)
        if waveform.max() > 1.0:
            waveform = waveform / 32768.0  # Convert from int16

        # Normalize
        waveform = librosa.util.normalize(waveform)

        # Extract MFCCs
        mfcc = librosa.feature.mfcc(y=waveform, sr=self.sample_rate, n_mfcc=self.mfcc_n_mfcc)
        delta = librosa.feature.delta(mfcc)
        delta2 = librosa.feature.delta(mfcc, order=2)

        # Concatenate: [13 + 13 + 13 = 39]
        features = np.vstack([mfcc, delta, delta2]).T  # [num_frames, 39]
        return sanitize_array(features.astype(np.float32))

    def extract_hubert(self, waveform: np.ndarray) -> np.ndarray:
        """Extract HuBERT embeddings.

        Args:
            waveform: Audio waveform [num_samples]

        Returns:
            HuBERT embeddings [num_frames, 768]
        """
        if not self.use_hubert:
            raise ValueError("HuBERT extraction not enabled")

        waveform = np.asarray(waveform, dtype=np.float32)
        if waveform.max() > 1.0:
            waveform = waveform / 32768.0

        with torch.no_grad():
            inputs = self._hubert_extractor(
                waveform,
                sampling_rate=self.sample_rate,
                return_tensors="pt",
                padding=True,
            )
            if torch.cuda.is_available():
                inputs = {k: v.cuda() for k, v in inputs.items()}
            
            outputs = self._hubert_model(**inputs)
            embeddings = outputs.last_hidden_state[0].cpu().numpy()  # [num_frames, 768]

        return sanitize_array(embeddings.astype(np.float32))

    def extract(self, waveform: np.ndarray) -> Dict[str, np.ndarray]:
        """Extract audio features.

        Args:
            waveform: Audio waveform [num_samples]

        Returns:
            Dict with 'mfcc' and optionally 'hubert' features
        """
        features = {}

        # MFCC is always extracted
        mfcc_features = self.extract_mfcc(waveform)
        if self.normalizer:
            mfcc_features = self.normalizer.transform(mfcc_features)
        features["mfcc"] = mfcc_features

        # HuBERT if enabled
        if self.use_hubert:
            hubert_features = self.extract_hubert(waveform)
            features["hubert"] = hubert_features

        return features


class TextFeatureRuntimeExtractor:
    """Extract text embeddings using SentenceTransformer.

    Produces training-compatible text embeddings.
    """

    def __init__(
        self,
        model_name: str = "sentence-transformers/all-mpnet-base-v2",
        device: str = "auto",
    ):
        """Initialize extractor.

        Args:
            model_name: HuggingFace model name
            device: Device to use ("auto", "cuda", "cpu")
        """
        if device == "auto":
            device = "cuda" if torch.cuda.is_available() else "cpu"
        
        self.device = device
        self.model = SentenceTransformer(model_name, device=device)

    def encode_text(self, text: str) -> np.ndarray:
        """Encode single text to embedding.

        Args:
            text: Text string

        Returns:
            Embedding [384]
        """
        embedding = self.model.encode(text, convert_to_numpy=True)
        return sanitize_array(embedding.astype(np.float32))

    def encode_chunks(self, chunks: Sequence[str]) -> np.ndarray:
        """Encode text chunks to embeddings.

        Args:
            chunks: List of text chunks

        Returns:
            Embeddings [num_chunks, 384]
        """
        embeddings = self.model.encode(
            list(chunks),
            convert_to_numpy=True,
            show_progress_bar=False,
        )
        return sanitize_array(embeddings.astype(np.float32))


class LiveInferenceFeatureExtractor:
    """Unified runtime feature extraction for live inference.

    Automatically produces training-compatible features from raw inputs.
    """

    def __init__(
        self,
        openface_pca: Optional[PCATransform] = None,
        resnet_pca: Optional[PCATransform] = None,
        audio_normalizer: Optional[FeatureNormalizer] = None,
        device: str = "auto",
    ):
        """Initialize extractor.

        Args:
            openface_pca: Optional PCA for OpenFace features
            resnet_pca: Optional PCA for ResNet embeddings
            audio_normalizer: Optional normalizer for audio features
            device: Device for model inference
        """
        self.openface_extractor = OpenFaceRuntimeExtractor(use_csv=True)
        self.resnet_extractor = ResNetRuntimeExtractor(
            device=device,
            pca_preprocessor=resnet_pca,
        )
        self.audio_extractor = AudioFeatureRuntimeExtractor(
            normalizer=audio_normalizer,
        )
        self.text_extractor = TextFeatureRuntimeExtractor(device=device)

    def extract_video(
        self,
        source: Path | str,
        use_openface_csv: bool = True,
    ) -> Dict[str, np.ndarray]:
        """Extract video features from file or CSV.

        Args:
            source: Path to video file or OpenFace CSV
            use_openface_csv: Whether source is OpenFace CSV

        Returns:
            Dict with 'openface' and optionally 'resnet' features
        """
        features = {}

        if use_openface_csv:
            features["openface"] = self.openface_extractor.extract(source)
        
        return features

    def extract_audio(self, audio_path: Path | str) -> Dict[str, np.ndarray]:
        """Extract audio features from file.

        Args:
            audio_path: Path to audio file

        Returns:
            Dict with 'mfcc' and optionally 'hubert' features
        """
        waveform, sr = librosa.load(str(audio_path), sr=self.audio_extractor.sample_rate, mono=True)
        return self.audio_extractor.extract(waveform)

    def extract_text(
        self,
        text: str,
        chunk_size: int = 256,
    ) -> np.ndarray:
        """Extract text embeddings.

        Args:
            text: Input text
            chunk_size: Approximate chunk size for splitting

        Returns:
            Embeddings [num_chunks, 384]
        """
        # Simple sentence-based chunking
        sentences = text.split(".")
        chunks = []
        current_chunk = ""
        
        for sentence in sentences:
            sentence = sentence.strip()
            if not sentence:
                continue
            if len(current_chunk.split()) + len(sentence.split()) <= chunk_size:
                current_chunk += " " + sentence if current_chunk else sentence
            else:
                if current_chunk:
                    chunks.append(current_chunk)
                current_chunk = sentence
        
        if current_chunk:
            chunks.append(current_chunk)

        if not chunks:
            chunks = [text]

        return self.text_extractor.encode_chunks(chunks)