koushik2109's picture
Model Changes
8d7b4b1
Raw History Blame Contribute Delete
8.05 kB
"""Feature specification and validation system for training/inference consistency.
This module defines strict feature specifications for each modality to ensure
training and inference produce identical feature representations.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Dict, Optional
import numpy as np
@dataclass
class ModalitySpec:
"""Specification for a single modality's features."""
name: str
expected_dim: int
temporal: bool = True
preprocessor_class: Optional[str] = None
extractor_class: Optional[str] = None
pca_enabled: bool = False
pca_target_dim: Optional[int] = None
normalizer_type: str = "standard" # "standard", "robust", "minmax", "none"
def validate_features(self, features: np.ndarray) -> bool:
"""Validate that features match specification."""
if not isinstance(features, np.ndarray):
return False
if features.dtype != np.float32:
return False
if self.temporal:
if features.ndim != 2:
return False
if features.shape[1] != self.expected_dim:
return False
else:
if features.ndim != 1:
return False
if features.shape[0] != self.expected_dim:
return False
return not (np.isnan(features).any() or np.isinf(features).any())
def to_dict(self) -> Dict:
"""Serialize spec to dict."""
return {
"name": self.name,
"expected_dim": self.expected_dim,
"temporal": self.temporal,
"preprocessor_class": self.preprocessor_class,
"extractor_class": self.extractor_class,
"pca_enabled": self.pca_enabled,
"pca_target_dim": self.pca_target_dim,
"normalizer_type": self.normalizer_type,
}
@classmethod
def from_dict(cls, data: Dict) -> ModalitySpec:
"""Deserialize spec from dict."""
return cls(**data)
class AudioSpec:
"""Audio modality specifications."""
# MFCC-based audio: 13 MFCC + 13 delta + 13 delta2 = 39
MFCC_DIM = 39
# HuBERT-base embeddings
HUBERT_DIM = 768
# Combined features: MFCC (39) + HuBERT (768) = 807
COMBINED_DIM = MFCC_DIM + HUBERT_DIM
# Standard specification: use MFCC features for training
STANDARD = ModalitySpec(
name="audio",
expected_dim=MFCC_DIM, # 39: 13 MFCC + 13 delta + 13 delta2
temporal=True,
preprocessor_class="AudioPreprocessor",
extractor_class="AudioFeatureExtractor",
pca_enabled=False,
normalizer_type="standard",
)
# Optional: HuBERT embeddings
HUBERT = ModalitySpec(
name="audio_hubert",
expected_dim=HUBERT_DIM, # 768
temporal=True,
preprocessor_class="AudioPreprocessor",
extractor_class="AudioFeatureExtractor",
pca_enabled=True,
pca_target_dim=128,
normalizer_type="standard",
)
# Optional: Combined MFCC + HuBERT
COMBINED = ModalitySpec(
name="audio_combined",
expected_dim=COMBINED_DIM, # 807
temporal=True,
preprocessor_class="AudioPreprocessor",
extractor_class="AudioFeatureExtractor",
pca_enabled=True,
pca_target_dim=128,
normalizer_type="standard",
)
class VideoSpec:
"""Video modality specifications."""
# OpenFace features: AU (17) + gaze (3) + head pose (3) + landmarks dynamics (many)
OPENFACE_DIM = 38 # Conservative estimate for behavioral features
# ResNet50 embeddings: 2048D, typically PCA-reduced
RESNET_RAW_DIM = 2048
RESNET_PCA_DIM = 128 # Standard reduction for ResNet
# Specification: behavioral features + visual embeddings
BEHAVIORAL = ModalitySpec(
name="video_behavior",
expected_dim=OPENFACE_DIM, # 38: OpenFace AU + gaze + pose
temporal=True,
preprocessor_class="VideoPreprocessor",
extractor_class="VideoFeatureExtractor",
pca_enabled=False,
normalizer_type="standard",
)
VISUAL_EMBEDDING = ModalitySpec(
name="video_embedding",
expected_dim=RESNET_PCA_DIM, # 128: ResNet50 PCA-reduced
temporal=True,
preprocessor_class="VideoPreprocessor",
extractor_class="VideoFeatureExtractor",
pca_enabled=True, # PCA is applied during extraction
pca_target_dim=RESNET_PCA_DIM,
normalizer_type="standard",
)
# Combined: behavioral + visual
COMBINED = ModalitySpec(
name="video_combined",
expected_dim=OPENFACE_DIM + RESNET_PCA_DIM, # 166: behavioral + visual
temporal=True,
preprocessor_class="VideoPreprocessor",
extractor_class="VideoFeatureExtractor",
pca_enabled=False,
normalizer_type="standard",
)
# Standard: behavioral features only (recommended for small datasets)
STANDARD = BEHAVIORAL
class TextSpec:
"""Text modality specifications."""
# SBERT all-mpnet-base-v2 embeddings
SBERT_DIM = 384
# Standard specification
STANDARD = ModalitySpec(
name="text",
expected_dim=SBERT_DIM, # 384: SBERT embeddings
temporal=True, # Multiple chunks per session
preprocessor_class="TextPreprocessor",
extractor_class="TextFeatureExtractor",
pca_enabled=False,
normalizer_type="standard",
)
class FeatureSpecification:
"""Complete multimodal feature specification for training/inference consistency."""
def __init__(
self,
audio_spec: ModalitySpec = AudioSpec.STANDARD,
video_spec: ModalitySpec = VideoSpec.STANDARD,
text_spec: ModalitySpec = TextSpec.STANDARD,
):
self.audio_spec = audio_spec
self.video_spec = video_spec
self.text_spec = text_spec
self._specs = {
"audio": audio_spec,
"video": video_spec,
"text": text_spec,
}
@property
def audio_dim(self) -> int:
return self.audio_spec.expected_dim
@property
def video_dim(self) -> int:
return self.video_spec.expected_dim
@property
def text_dim(self) -> int:
return self.text_spec.expected_dim
def get_spec(self, modality: str) -> ModalitySpec:
"""Get specification for a modality."""
if modality not in self._specs:
raise ValueError(f"Unknown modality: {modality}")
return self._specs[modality]
def validate(self, features: Dict[str, np.ndarray]) -> bool:
"""Validate all feature modalities match specification."""
for modality, feature_array in features.items():
if modality in self._specs:
if not self._specs[modality].validate_features(feature_array):
return False
return True
def to_dict(self) -> Dict:
"""Serialize specification."""
return {
"audio": self.audio_spec.to_dict(),
"video": self.video_spec.to_dict(),
"text": self.text_spec.to_dict(),
}
@classmethod
def default(cls) -> FeatureSpecification:
"""Create default specification: MFCC audio + behavioral video + SBERT text."""
return cls(
audio_spec=AudioSpec.STANDARD,
video_spec=VideoSpec.STANDARD,
text_spec=TextSpec.STANDARD,
)
@classmethod
def with_hubert(cls) -> FeatureSpecification:
"""Create specification with HuBERT audio embeddings."""
return cls(
audio_spec=AudioSpec.HUBERT,
video_spec=VideoSpec.STANDARD,
text_spec=TextSpec.STANDARD,
)
@classmethod
def multimodal_combined(cls) -> FeatureSpecification:
"""Create specification with all combined features."""
return cls(
audio_spec=AudioSpec.COMBINED,
video_spec=VideoSpec.COMBINED,
text_spec=TextSpec.STANDARD,
)