Spaces:
Sleeping
Sleeping
File size: 3,064 Bytes
8d7b4b1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | import math
from pathlib import Path
from typing import Dict, Optional
import librosa
import numpy as np
import torch
from transformers import Wav2Vec2FeatureExtractor, Wav2Vec2Model
from src.features.sanitization import sanitize_array
class AudioFeatureExtractor:
"""Audio feature extraction for train and inference.
Supports classical features (MFCC, delta, silence ratio) and
modern self-supervised embeddings (HuBERT/Wav2Vec2).
"""
def __init__(self, config: dict):
self.sample_rate = config.get("sample_rate", 16000)
self.mfcc_n_mfcc = config.get("mfcc_n_mfcc", 13)
self.hubert_model_name = config.get("hubert_model_name", "facebook/hubert-base-ls960")
self.feature_backends = tuple(config.get("feature_backends", ["mfcc", "hubert"]))
self._hubert_extractor: Optional[Wav2Vec2FeatureExtractor] = None
self._hubert_model: Optional[Wav2Vec2Model] = None
def _ensure_hubert(self):
if self._hubert_model is None:
self._hubert_extractor = Wav2Vec2FeatureExtractor.from_pretrained(self.hubert_model_name)
self._hubert_model = Wav2Vec2Model.from_pretrained(self.hubert_model_name)
self._hubert_model.eval()
def extract_classical(self, waveform: np.ndarray) -> Dict[str, np.ndarray]:
waveform = waveform.astype(np.float32)
waveform = librosa.util.normalize(waveform)
mfcc = librosa.feature.mfcc(y=waveform, sr=self.sample_rate, n_mfcc=self.mfcc_n_mfcc)
delta = librosa.feature.delta(mfcc)
delta2 = librosa.feature.delta(mfcc, order=2)
energy = librosa.feature.rms(y=waveform)[0]
silence_ratio = float(np.mean(energy < np.percentile(energy, 10)))
speech_rate = float(np.sum(energy > np.median(energy)) / max(1, len(energy)))
return {
"mfcc": sanitize_array(mfcc.T),
"delta": sanitize_array(delta.T),
"delta2": sanitize_array(delta2.T),
"silence_ratio": np.array([silence_ratio], dtype=np.float32),
"speech_rate": np.array([speech_rate], dtype=np.float32),
}
def extract_modern(self, waveform: np.ndarray) -> np.ndarray:
self._ensure_hubert()
waveform = waveform.astype(np.float32)
if waveform.ndim > 1:
waveform = waveform.mean(axis=-1)
inputs = self._hubert_extractor(waveform, sampling_rate=self.sample_rate, return_tensors="pt", padding=True)
with torch.no_grad():
outputs = self._hubert_model(**inputs)
embedding = outputs.last_hidden_state.mean(dim=1)
return sanitize_array(embedding.squeeze(0).cpu().numpy())
def extract_features(self, waveform: np.ndarray) -> Dict[str, np.ndarray]:
features: Dict[str, np.ndarray] = {}
if "mfcc" in self.feature_backends:
classical = self.extract_classical(waveform)
features.update(classical)
if "hubert" in self.feature_backends:
features["hubert"] = self.extract_modern(waveform)
return features
|