Spaces:
Sleeping
Sleeping
Download src/features/specs.py from koushik2109/mindscope-Model: direct link, hf CLI and curl.
- Browser
- Download file 8.05 kB
-
https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/features/specs.py
- Command line
-
hf download hf://spaces/koushik2109/mindscope-Model/src/features/specs.py
-
curl -L -o specs.py https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/features/specs.py
8.05 kB
| """Feature specification and validation system for training/inference consistency. | |
| This module defines strict feature specifications for each modality to ensure | |
| training and inference produce identical feature representations. | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import dataclass | |
| from typing import Dict, Optional | |
| import numpy as np | |
| class ModalitySpec: | |
| """Specification for a single modality's features.""" | |
| name: str | |
| expected_dim: int | |
| temporal: bool = True | |
| preprocessor_class: Optional[str] = None | |
| extractor_class: Optional[str] = None | |
| pca_enabled: bool = False | |
| pca_target_dim: Optional[int] = None | |
| normalizer_type: str = "standard" # "standard", "robust", "minmax", "none" | |
| def validate_features(self, features: np.ndarray) -> bool: | |
| """Validate that features match specification.""" | |
| if not isinstance(features, np.ndarray): | |
| return False | |
| if features.dtype != np.float32: | |
| return False | |
| if self.temporal: | |
| if features.ndim != 2: | |
| return False | |
| if features.shape[1] != self.expected_dim: | |
| return False | |
| else: | |
| if features.ndim != 1: | |
| return False | |
| if features.shape[0] != self.expected_dim: | |
| return False | |
| return not (np.isnan(features).any() or np.isinf(features).any()) | |
| def to_dict(self) -> Dict: | |
| """Serialize spec to dict.""" | |
| return { | |
| "name": self.name, | |
| "expected_dim": self.expected_dim, | |
| "temporal": self.temporal, | |
| "preprocessor_class": self.preprocessor_class, | |
| "extractor_class": self.extractor_class, | |
| "pca_enabled": self.pca_enabled, | |
| "pca_target_dim": self.pca_target_dim, | |
| "normalizer_type": self.normalizer_type, | |
| } | |
| def from_dict(cls, data: Dict) -> ModalitySpec: | |
| """Deserialize spec from dict.""" | |
| return cls(**data) | |
| class AudioSpec: | |
| """Audio modality specifications.""" | |
| # MFCC-based audio: 13 MFCC + 13 delta + 13 delta2 = 39 | |
| MFCC_DIM = 39 | |
| # HuBERT-base embeddings | |
| HUBERT_DIM = 768 | |
| # Combined features: MFCC (39) + HuBERT (768) = 807 | |
| COMBINED_DIM = MFCC_DIM + HUBERT_DIM | |
| # Standard specification: use MFCC features for training | |
| STANDARD = ModalitySpec( | |
| name="audio", | |
| expected_dim=MFCC_DIM, # 39: 13 MFCC + 13 delta + 13 delta2 | |
| temporal=True, | |
| preprocessor_class="AudioPreprocessor", | |
| extractor_class="AudioFeatureExtractor", | |
| pca_enabled=False, | |
| normalizer_type="standard", | |
| ) | |
| # Optional: HuBERT embeddings | |
| HUBERT = ModalitySpec( | |
| name="audio_hubert", | |
| expected_dim=HUBERT_DIM, # 768 | |
| temporal=True, | |
| preprocessor_class="AudioPreprocessor", | |
| extractor_class="AudioFeatureExtractor", | |
| pca_enabled=True, | |
| pca_target_dim=128, | |
| normalizer_type="standard", | |
| ) | |
| # Optional: Combined MFCC + HuBERT | |
| COMBINED = ModalitySpec( | |
| name="audio_combined", | |
| expected_dim=COMBINED_DIM, # 807 | |
| temporal=True, | |
| preprocessor_class="AudioPreprocessor", | |
| extractor_class="AudioFeatureExtractor", | |
| pca_enabled=True, | |
| pca_target_dim=128, | |
| normalizer_type="standard", | |
| ) | |
| class VideoSpec: | |
| """Video modality specifications.""" | |
| # OpenFace features: AU (17) + gaze (3) + head pose (3) + landmarks dynamics (many) | |
| OPENFACE_DIM = 38 # Conservative estimate for behavioral features | |
| # ResNet50 embeddings: 2048D, typically PCA-reduced | |
| RESNET_RAW_DIM = 2048 | |
| RESNET_PCA_DIM = 128 # Standard reduction for ResNet | |
| # Specification: behavioral features + visual embeddings | |
| BEHAVIORAL = ModalitySpec( | |
| name="video_behavior", | |
| expected_dim=OPENFACE_DIM, # 38: OpenFace AU + gaze + pose | |
| temporal=True, | |
| preprocessor_class="VideoPreprocessor", | |
| extractor_class="VideoFeatureExtractor", | |
| pca_enabled=False, | |
| normalizer_type="standard", | |
| ) | |
| VISUAL_EMBEDDING = ModalitySpec( | |
| name="video_embedding", | |
| expected_dim=RESNET_PCA_DIM, # 128: ResNet50 PCA-reduced | |
| temporal=True, | |
| preprocessor_class="VideoPreprocessor", | |
| extractor_class="VideoFeatureExtractor", | |
| pca_enabled=True, # PCA is applied during extraction | |
| pca_target_dim=RESNET_PCA_DIM, | |
| normalizer_type="standard", | |
| ) | |
| # Combined: behavioral + visual | |
| COMBINED = ModalitySpec( | |
| name="video_combined", | |
| expected_dim=OPENFACE_DIM + RESNET_PCA_DIM, # 166: behavioral + visual | |
| temporal=True, | |
| preprocessor_class="VideoPreprocessor", | |
| extractor_class="VideoFeatureExtractor", | |
| pca_enabled=False, | |
| normalizer_type="standard", | |
| ) | |
| # Standard: behavioral features only (recommended for small datasets) | |
| STANDARD = BEHAVIORAL | |
| class TextSpec: | |
| """Text modality specifications.""" | |
| # SBERT all-mpnet-base-v2 embeddings | |
| SBERT_DIM = 384 | |
| # Standard specification | |
| STANDARD = ModalitySpec( | |
| name="text", | |
| expected_dim=SBERT_DIM, # 384: SBERT embeddings | |
| temporal=True, # Multiple chunks per session | |
| preprocessor_class="TextPreprocessor", | |
| extractor_class="TextFeatureExtractor", | |
| pca_enabled=False, | |
| normalizer_type="standard", | |
| ) | |
| class FeatureSpecification: | |
| """Complete multimodal feature specification for training/inference consistency.""" | |
| def __init__( | |
| self, | |
| audio_spec: ModalitySpec = AudioSpec.STANDARD, | |
| video_spec: ModalitySpec = VideoSpec.STANDARD, | |
| text_spec: ModalitySpec = TextSpec.STANDARD, | |
| ): | |
| self.audio_spec = audio_spec | |
| self.video_spec = video_spec | |
| self.text_spec = text_spec | |
| self._specs = { | |
| "audio": audio_spec, | |
| "video": video_spec, | |
| "text": text_spec, | |
| } | |
| def audio_dim(self) -> int: | |
| return self.audio_spec.expected_dim | |
| def video_dim(self) -> int: | |
| return self.video_spec.expected_dim | |
| def text_dim(self) -> int: | |
| return self.text_spec.expected_dim | |
| def get_spec(self, modality: str) -> ModalitySpec: | |
| """Get specification for a modality.""" | |
| if modality not in self._specs: | |
| raise ValueError(f"Unknown modality: {modality}") | |
| return self._specs[modality] | |
| def validate(self, features: Dict[str, np.ndarray]) -> bool: | |
| """Validate all feature modalities match specification.""" | |
| for modality, feature_array in features.items(): | |
| if modality in self._specs: | |
| if not self._specs[modality].validate_features(feature_array): | |
| return False | |
| return True | |
| def to_dict(self) -> Dict: | |
| """Serialize specification.""" | |
| return { | |
| "audio": self.audio_spec.to_dict(), | |
| "video": self.video_spec.to_dict(), | |
| "text": self.text_spec.to_dict(), | |
| } | |
| def default(cls) -> FeatureSpecification: | |
| """Create default specification: MFCC audio + behavioral video + SBERT text.""" | |
| return cls( | |
| audio_spec=AudioSpec.STANDARD, | |
| video_spec=VideoSpec.STANDARD, | |
| text_spec=TextSpec.STANDARD, | |
| ) | |
| def with_hubert(cls) -> FeatureSpecification: | |
| """Create specification with HuBERT audio embeddings.""" | |
| return cls( | |
| audio_spec=AudioSpec.HUBERT, | |
| video_spec=VideoSpec.STANDARD, | |
| text_spec=TextSpec.STANDARD, | |
| ) | |
| def multimodal_combined(cls) -> FeatureSpecification: | |
| """Create specification with all combined features.""" | |
| return cls( | |
| audio_spec=AudioSpec.COMBINED, | |
| video_spec=VideoSpec.COMBINED, | |
| text_spec=TextSpec.STANDARD, | |
| ) | |