Spaces:
Sleeping
Sleeping
File size: 2,368 Bytes
8d7b4b1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | import argparse
from pathlib import Path
from typing import Dict, Any
import yaml
from src.features.audio_features import AudioFeatureExtractor
from src.features.video_features import VideoFeatureExtractor
from src.features.text_features import TextFeatureExtractor
from src.features.feature_store import save_features
from src.preprocessing.audio_preprocessor import AudioPreprocessor
from src.preprocessing.text_preprocessor import TextPreprocessor
def load_config(path: Path) -> Dict[str, Any]:
with open(path, "r", encoding="utf-8") as handle:
return yaml.safe_load(handle)
def main() -> None:
parser = argparse.ArgumentParser(description="Extract ModelV2 multimodal features")
parser.add_argument("--config", type=Path, default=Path("configs/feature_extraction.yaml"))
parser.add_argument("--input-dir", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
args = parser.parse_args()
config = load_config(args.config)
audio_preprocessor = AudioPreprocessor(**config["audio"])
audio_extractor = AudioFeatureExtractor(config["audio"])
video_extractor = VideoFeatureExtractor(config["video"])
text_preprocessor = TextPreprocessor()
text_extractor = TextFeatureExtractor(config["text"]["transformer_model"])
for subject_dir in args.input_dir.iterdir():
if not subject_dir.is_dir():
continue
participant_id = subject_dir.name
audio_path = subject_dir / "audio.wav"
transcript_path = subject_dir / "transcript.txt"
openface_path = subject_dir / "openface.csv"
if not audio_path.exists() or not transcript_path.exists() or not openface_path.exists():
continue
waveform = audio_preprocessor.load_audio(audio_path)
audio_chunks = audio_preprocessor.preprocess(waveform)
audio_features = {"audio": audio_extractor.extract_modern(waveform)}
video_behavior = video_extractor.extract_openface(openface_path)["behavior"]
text = transcript_path.read_text(encoding="utf-8")
chunks = text_preprocessor.preprocess(text)
text_features = {"text": text_extractor.encode(chunks)}
save_features(args.output_dir / f"{participant_id}.npz", {
**audio_features,
"video": video_behavior,
**text_features,
})
|