Spaces:
Sleeping
Sleeping
Download src/features/audio_features.py from koushik2109/mindscope-Model: direct link, hf CLI and curl.
- Browser
- Download file 3.06 kB
-
https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/features/audio_features.py
- Command line
-
hf download hf://spaces/koushik2109/mindscope-Model/src/features/audio_features.py
-
curl -L -o audio_features.py https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/features/audio_features.py
3.06 kB
| import math | |
| from pathlib import Path | |
| from typing import Dict, Optional | |
| import librosa | |
| import numpy as np | |
| import torch | |
| from transformers import Wav2Vec2FeatureExtractor, Wav2Vec2Model | |
| from src.features.sanitization import sanitize_array | |
| class AudioFeatureExtractor: | |
| """Audio feature extraction for train and inference. | |
| Supports classical features (MFCC, delta, silence ratio) and | |
| modern self-supervised embeddings (HuBERT/Wav2Vec2). | |
| """ | |
| def __init__(self, config: dict): | |
| self.sample_rate = config.get("sample_rate", 16000) | |
| self.mfcc_n_mfcc = config.get("mfcc_n_mfcc", 13) | |
| self.hubert_model_name = config.get("hubert_model_name", "facebook/hubert-base-ls960") | |
| self.feature_backends = tuple(config.get("feature_backends", ["mfcc", "hubert"])) | |
| self._hubert_extractor: Optional[Wav2Vec2FeatureExtractor] = None | |
| self._hubert_model: Optional[Wav2Vec2Model] = None | |
| def _ensure_hubert(self): | |
| if self._hubert_model is None: | |
| self._hubert_extractor = Wav2Vec2FeatureExtractor.from_pretrained(self.hubert_model_name) | |
| self._hubert_model = Wav2Vec2Model.from_pretrained(self.hubert_model_name) | |
| self._hubert_model.eval() | |
| def extract_classical(self, waveform: np.ndarray) -> Dict[str, np.ndarray]: | |
| waveform = waveform.astype(np.float32) | |
| waveform = librosa.util.normalize(waveform) | |
| mfcc = librosa.feature.mfcc(y=waveform, sr=self.sample_rate, n_mfcc=self.mfcc_n_mfcc) | |
| delta = librosa.feature.delta(mfcc) | |
| delta2 = librosa.feature.delta(mfcc, order=2) | |
| energy = librosa.feature.rms(y=waveform)[0] | |
| silence_ratio = float(np.mean(energy < np.percentile(energy, 10))) | |
| speech_rate = float(np.sum(energy > np.median(energy)) / max(1, len(energy))) | |
| return { | |
| "mfcc": sanitize_array(mfcc.T), | |
| "delta": sanitize_array(delta.T), | |
| "delta2": sanitize_array(delta2.T), | |
| "silence_ratio": np.array([silence_ratio], dtype=np.float32), | |
| "speech_rate": np.array([speech_rate], dtype=np.float32), | |
| } | |
| def extract_modern(self, waveform: np.ndarray) -> np.ndarray: | |
| self._ensure_hubert() | |
| waveform = waveform.astype(np.float32) | |
| if waveform.ndim > 1: | |
| waveform = waveform.mean(axis=-1) | |
| inputs = self._hubert_extractor(waveform, sampling_rate=self.sample_rate, return_tensors="pt", padding=True) | |
| with torch.no_grad(): | |
| outputs = self._hubert_model(**inputs) | |
| embedding = outputs.last_hidden_state.mean(dim=1) | |
| return sanitize_array(embedding.squeeze(0).cpu().numpy()) | |
| def extract_features(self, waveform: np.ndarray) -> Dict[str, np.ndarray]: | |
| features: Dict[str, np.ndarray] = {} | |
| if "mfcc" in self.feature_backends: | |
| classical = self.extract_classical(waveform) | |
| features.update(classical) | |
| if "hubert" in self.feature_backends: | |
| features["hubert"] = self.extract_modern(waveform) | |
| return features | |