mindscope-Model / src /inference /runtime_extractors.py
koushik2109's picture
Model Changes
8d7b4b1
Raw History Blame Contribute Delete
14.8 kB
"""Modular runtime feature extractors for live inference.
These extractors automatically generate training-compatible features from raw
webcam/video/audio input without manual configuration.
"""
from __future__ import annotations
import csv
from pathlib import Path
from typing import Dict, List, Optional, Sequence
import numpy as np
import torch
import librosa
import torchvision.models as models
import torchvision.transforms as T
from transformers import Wav2Vec2FeatureExtractor, Wav2Vec2Model
from sentence_transformers import SentenceTransformer
from src.features.sanitization import sanitize_array
from src.features.preprocessing import FeaturePreprocessor, PCATransform, FeatureNormalizer
from src.features.specs import AudioSpec, VideoSpec, TextSpec
class OpenFaceRuntimeExtractor:
"""Extract OpenFace behavioral features from video frames or CSV.
Supports:
- Direct frame-based extraction (requires OpenFace installed)
- CSV loading for pre-computed OpenFace features
"""
EXPECTED_COLUMNS = [
"AU01_r", "AU02_r", "AU04_r", "AU05_r", "AU06_r", "AU07_r", "AU09_r",
"AU10_r", "AU12_r", "AU14_r", "AU15_r", "AU17_r", "AU20_r", "AU23_r",
"AU25_r", "AU26_r", "AU45_r", # 17 AUs
"Gaze_angle_x", "Gaze_angle_y", "Gaze_angle_z", # 3 gaze angles
"Pose_Tx", "Pose_Ty", "Pose_Tz", # 3 head pose
]
def __init__(self, use_csv: bool = True):
"""Initialize extractor.
Args:
use_csv: If True, load from CSV. If False, use frame-based extraction.
"""
self.use_csv = use_csv
def extract_from_csv(self, csv_path: Path) -> np.ndarray:
"""Extract OpenFace features from CSV file.
Args:
csv_path: Path to OpenFace CSV file
Returns:
Features array [num_frames, num_features]
"""
if not csv_path.exists():
raise FileNotFoundError(f"OpenFace CSV not found: {csv_path}")
with open(csv_path, "r", encoding="utf-8") as f:
reader = csv.DictReader(f)
rows = list(reader)
if not rows:
raise ValueError(f"OpenFace CSV is empty: {csv_path}")
# Extract features
features_list = []
for row in rows:
try:
# Extract AU features
au_features = [float(row.get(col, 0.0)) for col in self.EXPECTED_COLUMNS[:17]]
# Extract gaze
gaze_features = [float(row.get(col, 0.0)) for col in self.EXPECTED_COLUMNS[17:20]]
# Extract head pose
pose_features = [float(row.get(col, 0.0)) for col in self.EXPECTED_COLUMNS[20:23]]
features_list.append(au_features + gaze_features + pose_features)
except (ValueError, KeyError) as e:
continue
if not features_list:
raise ValueError(f"Could not extract valid features from {csv_path}")
features = np.array(features_list, dtype=np.float32)
return sanitize_array(features)
def extract(self, source: Path | str) -> np.ndarray:
"""Extract OpenFace features from CSV or video frames."""
source_path = Path(source) if isinstance(source, str) else source
if source_path.suffix == ".csv":
return self.extract_from_csv(source_path)
else:
raise NotImplementedError(
"Frame-based OpenFace extraction requires additional dependencies. "
"Use CSV format for now."
)
class ResNetRuntimeExtractor:
"""Extract visual embeddings from video frames using ResNet50.
Supports:
- Frame-by-frame extraction
- Temporal aggregation
- PCA reduction for consistency with training
"""
def __init__(
self,
device: str = "auto",
pca_preprocessor: Optional[FeaturePreprocessor] = None,
):
"""Initialize extractor.
Args:
device: Device to use ("auto", "cuda", "cpu")
pca_preprocessor: Optional PCA preprocessor for dimensionality reduction
"""
if device == "auto":
device = "cuda" if torch.cuda.is_available() else "cpu"
self.device = torch.device(device)
self.pca_preprocessor = pca_preprocessor
# Load pretrained ResNet50
model = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V1)
self.backbone = torch.nn.Sequential(*list(model.children())[:-1]).to(self.device)
self.backbone.eval()
# Image preprocessing
self.transform = T.Compose([
T.Resize((224, 224)),
T.ToTensor(),
T.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
),
])
def extract_from_frames(self, frames: Sequence[np.ndarray]) -> np.ndarray:
"""Extract embeddings from video frames.
Args:
frames: List of frames as np.ndarray [H, W, 3]
Returns:
Embeddings array [num_frames, embedding_dim]
"""
if not frames:
raise ValueError("No frames provided")
embeddings_list = []
with torch.no_grad():
for frame in frames:
if isinstance(frame, np.ndarray):
# Convert BGR to RGB if needed
if frame.shape[2] == 3:
if frame.max() > 1.0: # Likely 0-255 range
frame = frame.astype(np.float32) / 255.0
frame_tensor = torch.from_numpy(frame).permute(2, 0, 1).float()
else:
frame_tensor = self.transform(T.ToPILImage()(frame))
else:
frame_tensor = self.transform(frame)
frame_tensor = frame_tensor.unsqueeze(0).to(self.device)
embedding = self.backbone(frame_tensor).squeeze()
embeddings_list.append(embedding.cpu().numpy())
embeddings = np.array(embeddings_list, dtype=np.float32) # [num_frames, 2048]
embeddings = sanitize_array(embeddings)
# Apply PCA if available
if self.pca_preprocessor:
embeddings = self.pca_preprocessor.transform(embeddings)
return embeddings
class AudioFeatureRuntimeExtractor:
"""Extract audio features: MFCC + HuBERT.
Produces training-compatible audio features.
"""
def __init__(
self,
sample_rate: int = 16000,
mfcc_n_mfcc: int = 13,
use_hubert: bool = False,
normalizer: Optional[FeatureNormalizer] = None,
):
"""Initialize extractor.
Args:
sample_rate: Audio sample rate
mfcc_n_mfcc: Number of MFCC coefficients
use_hubert: Whether to include HuBERT embeddings
normalizer: Optional normalizer for features
"""
self.sample_rate = sample_rate
self.mfcc_n_mfcc = mfcc_n_mfcc
self.use_hubert = use_hubert
self.normalizer = normalizer
if use_hubert:
self._hubert_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
"facebook/hubert-base-ls960"
)
self._hubert_model = Wav2Vec2Model.from_pretrained("facebook/hubert-base-ls960")
self._hubert_model.eval()
if torch.cuda.is_available():
self._hubert_model = self._hubert_model.cuda()
def extract_mfcc(self, waveform: np.ndarray) -> np.ndarray:
"""Extract MFCC features: 13 MFCC + 13 delta + 13 delta-delta.
Args:
waveform: Audio waveform [num_samples]
Returns:
MFCC features [num_frames, 39]
"""
waveform = np.asarray(waveform, dtype=np.float32)
if waveform.max() > 1.0:
waveform = waveform / 32768.0 # Convert from int16
# Normalize
waveform = librosa.util.normalize(waveform)
# Extract MFCCs
mfcc = librosa.feature.mfcc(y=waveform, sr=self.sample_rate, n_mfcc=self.mfcc_n_mfcc)
delta = librosa.feature.delta(mfcc)
delta2 = librosa.feature.delta(mfcc, order=2)
# Concatenate: [13 + 13 + 13 = 39]
features = np.vstack([mfcc, delta, delta2]).T # [num_frames, 39]
return sanitize_array(features.astype(np.float32))
def extract_hubert(self, waveform: np.ndarray) -> np.ndarray:
"""Extract HuBERT embeddings.
Args:
waveform: Audio waveform [num_samples]
Returns:
HuBERT embeddings [num_frames, 768]
"""
if not self.use_hubert:
raise ValueError("HuBERT extraction not enabled")
waveform = np.asarray(waveform, dtype=np.float32)
if waveform.max() > 1.0:
waveform = waveform / 32768.0
with torch.no_grad():
inputs = self._hubert_extractor(
waveform,
sampling_rate=self.sample_rate,
return_tensors="pt",
padding=True,
)
if torch.cuda.is_available():
inputs = {k: v.cuda() for k, v in inputs.items()}
outputs = self._hubert_model(**inputs)
embeddings = outputs.last_hidden_state[0].cpu().numpy() # [num_frames, 768]
return sanitize_array(embeddings.astype(np.float32))
def extract(self, waveform: np.ndarray) -> Dict[str, np.ndarray]:
"""Extract audio features.
Args:
waveform: Audio waveform [num_samples]
Returns:
Dict with 'mfcc' and optionally 'hubert' features
"""
features = {}
# MFCC is always extracted
mfcc_features = self.extract_mfcc(waveform)
if self.normalizer:
mfcc_features = self.normalizer.transform(mfcc_features)
features["mfcc"] = mfcc_features
# HuBERT if enabled
if self.use_hubert:
hubert_features = self.extract_hubert(waveform)
features["hubert"] = hubert_features
return features
class TextFeatureRuntimeExtractor:
"""Extract text embeddings using SentenceTransformer.
Produces training-compatible text embeddings.
"""
def __init__(
self,
model_name: str = "sentence-transformers/all-mpnet-base-v2",
device: str = "auto",
):
"""Initialize extractor.
Args:
model_name: HuggingFace model name
device: Device to use ("auto", "cuda", "cpu")
"""
if device == "auto":
device = "cuda" if torch.cuda.is_available() else "cpu"
self.device = device
self.model = SentenceTransformer(model_name, device=device)
def encode_text(self, text: str) -> np.ndarray:
"""Encode single text to embedding.
Args:
text: Text string
Returns:
Embedding [384]
"""
embedding = self.model.encode(text, convert_to_numpy=True)
return sanitize_array(embedding.astype(np.float32))
def encode_chunks(self, chunks: Sequence[str]) -> np.ndarray:
"""Encode text chunks to embeddings.
Args:
chunks: List of text chunks
Returns:
Embeddings [num_chunks, 384]
"""
embeddings = self.model.encode(
list(chunks),
convert_to_numpy=True,
show_progress_bar=False,
)
return sanitize_array(embeddings.astype(np.float32))
class LiveInferenceFeatureExtractor:
"""Unified runtime feature extraction for live inference.
Automatically produces training-compatible features from raw inputs.
"""
def __init__(
self,
openface_pca: Optional[PCATransform] = None,
resnet_pca: Optional[PCATransform] = None,
audio_normalizer: Optional[FeatureNormalizer] = None,
device: str = "auto",
):
"""Initialize extractor.
Args:
openface_pca: Optional PCA for OpenFace features
resnet_pca: Optional PCA for ResNet embeddings
audio_normalizer: Optional normalizer for audio features
device: Device for model inference
"""
self.openface_extractor = OpenFaceRuntimeExtractor(use_csv=True)
self.resnet_extractor = ResNetRuntimeExtractor(
device=device,
pca_preprocessor=resnet_pca,
)
self.audio_extractor = AudioFeatureRuntimeExtractor(
normalizer=audio_normalizer,
)
self.text_extractor = TextFeatureRuntimeExtractor(device=device)
def extract_video(
self,
source: Path | str,
use_openface_csv: bool = True,
) -> Dict[str, np.ndarray]:
"""Extract video features from file or CSV.
Args:
source: Path to video file or OpenFace CSV
use_openface_csv: Whether source is OpenFace CSV
Returns:
Dict with 'openface' and optionally 'resnet' features
"""
features = {}
if use_openface_csv:
features["openface"] = self.openface_extractor.extract(source)
return features
def extract_audio(self, audio_path: Path | str) -> Dict[str, np.ndarray]:
"""Extract audio features from file.
Args:
audio_path: Path to audio file
Returns:
Dict with 'mfcc' and optionally 'hubert' features
"""
waveform, sr = librosa.load(str(audio_path), sr=self.audio_extractor.sample_rate, mono=True)
return self.audio_extractor.extract(waveform)
def extract_text(
self,
text: str,
chunk_size: int = 256,
) -> np.ndarray:
"""Extract text embeddings.
Args:
text: Input text
chunk_size: Approximate chunk size for splitting
Returns:
Embeddings [num_chunks, 384]
"""
# Simple sentence-based chunking
sentences = text.split(".")
chunks = []
current_chunk = ""
for sentence in sentences:
sentence = sentence.strip()
if not sentence:
continue
if len(current_chunk.split()) + len(sentence.split()) <= chunk_size:
current_chunk += " " + sentence if current_chunk else sentence
else:
if current_chunk:
chunks.append(current_chunk)
current_chunk = sentence
if current_chunk:
chunks.append(current_chunk)
if not chunks:
chunks = [text]
return self.text_extractor.encode_chunks(chunks)