diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..7c7546243650dc4bc4bfdcd9599ad0639c8306a9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,28 @@ +.venv/ +.git/ +__pycache__/ +*.pyc +*.pyo +logs*/ +logs_v4_run.log +data/ +notebooks/ +docs/ +runs/ +*.zip +*.tar.gz +checkpoints/cv/ +checkpoints/multimodal_v2/ +checkpoints/multimodal_v3/ +checkpoints/best_fusion.pt +checkpoints/best_model_backup_0.14ccc.pt +checkpoints/best_stage1.pt +checkpoints/best_stage2_clean.pt +checkpoints/best_trimodal.pt +checkpoints/best_trimodal_backup.pt +checkpoints/best_trimodal_text.pt +checkpoints/pca_cnn_transform.pkl +checkpoints/scalers/ +windows/ +.vscode/ +tests/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..5282617085af0e3bca854c1288e23e18f5e83e60 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.11-slim + +WORKDIR /app + +# System deps: libgl (mediapipe/opencv), ffmpeg (audio), libgomp (torch) +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgl1 \ + libglib2.0-0 \ + libsm6 \ + libxext6 \ + libxrender-dev \ + libgomp1 \ + ffmpeg \ + git \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . + +# Install CPU-only torch — latest from wheel index (no version pin) +# Same approach as start-all.sh: installs latest compatible cpu set +RUN pip install --no-cache-dir \ + torch torchaudio torchvision \ + --index-url https://download.pytorch.org/whl/cpu + +# Install remaining deps (torch already satisfied above, won't re-download) +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +EXPOSE 8001 + +CMD ["python", "scripts/serve.py", "--port", "8001"] diff --git a/checkpoints/best_fusion.pt b/checkpoints/best_fusion.pt new file mode 100644 index 0000000000000000000000000000000000000000..b70184c878d55f9a28c1b258a288a558afa8c6f0 --- /dev/null +++ b/checkpoints/best_fusion.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a3f27f622fcf180f1704d4612273a97813a85d8893b87e9a185d2915cdcbeda +size 43669 diff --git a/checkpoints/best_model.pt b/checkpoints/best_model.pt new file mode 100644 index 0000000000000000000000000000000000000000..63465c4a7cb53bf7e20e82e7a797a8f3867f5137 --- /dev/null +++ b/checkpoints/best_model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:147924ec16e02220048ff2c28ad71daafe3a93d35771ca9e4e4de1279c4e78c3 +size 405941 diff --git a/checkpoints/best_model_backup_0.14ccc.pt b/checkpoints/best_model_backup_0.14ccc.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ed86577938be352935225112be89596dcb0f800 --- /dev/null +++ b/checkpoints/best_model_backup_0.14ccc.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f0ed9844a410329a275b9fc7bbdc3e89128ebeb292e568ae6695d329936177d +size 43125 diff --git a/checkpoints/best_stage1.pt b/checkpoints/best_stage1.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff1aef1395840ddbb04e7e5baa1043e7e7a93fae --- /dev/null +++ b/checkpoints/best_stage1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc5946d5372483f4eaeadba08255ce16eaa7bf656b7ca15a0c5a8d12d1ea3551 +size 349073 diff --git a/checkpoints/best_stage2_clean.pt b/checkpoints/best_stage2_clean.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a65ac8de535d322af86582229e53ae276fe1ad1 --- /dev/null +++ b/checkpoints/best_stage2_clean.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b966db1077d73ce7e88a8e212b3d16c3f5994d8d128007e69f944f6adf82c505 +size 1870025 diff --git a/checkpoints/best_trimodal.pt b/checkpoints/best_trimodal.pt new file mode 100644 index 0000000000000000000000000000000000000000..23dd72e9d4805f0ca325e3581cc89a5f39692368 --- /dev/null +++ b/checkpoints/best_trimodal.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c50f63feb8adcce165846a833ac72bf6601c2d96624b24b71d12ced38627b6f +size 1869605 diff --git a/checkpoints/best_trimodal_backup.pt b/checkpoints/best_trimodal_backup.pt new file mode 100644 index 0000000000000000000000000000000000000000..6bae97d4ad008a9c96a5cd9cf711e667638b86a7 --- /dev/null +++ b/checkpoints/best_trimodal_backup.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39ab55d17b81e93f47cbf988a190fd26e8f6a83570f366321a4daf1835f4d046 +size 1869255 diff --git a/checkpoints/best_trimodal_text.pt b/checkpoints/best_trimodal_text.pt new file mode 100644 index 0000000000000000000000000000000000000000..2df090b5a9f34dec8e8ae0b78adaf2cac0985952 --- /dev/null +++ b/checkpoints/best_trimodal_text.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:818cccc4fab056005b651ead9ef7bb5c430be39b31a291d3928a9002e9813405 +size 1891597 diff --git a/checkpoints/cv/best_model_fold0.pt b/checkpoints/cv/best_model_fold0.pt new file mode 100644 index 0000000000000000000000000000000000000000..964e2746b5f2417f3ada362087a3ea2e46d723ea --- /dev/null +++ b/checkpoints/cv/best_model_fold0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2783e75575e344833fbc9224a7191e2be957cdc6d6f3016378618b11ac6bc930 +size 43339 diff --git a/checkpoints/cv/best_model_fold1.pt b/checkpoints/cv/best_model_fold1.pt new file mode 100644 index 0000000000000000000000000000000000000000..98322e40e5a8a6646aeef296163695eb1061c416 --- /dev/null +++ b/checkpoints/cv/best_model_fold1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1b18a628524c2aa70398976ed90d36131ffbe7910675a5792f17540f244ea77 +size 43339 diff --git a/checkpoints/cv/best_model_fold2.pt b/checkpoints/cv/best_model_fold2.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f258e5573aa516c2fb860d5a05ce10fc99134fc --- /dev/null +++ b/checkpoints/cv/best_model_fold2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f616da338d0dadba40884826c4959d47afeae3a321ffc907445fa6afcb8d7e4 +size 43339 diff --git a/checkpoints/cv/best_model_fold3.pt b/checkpoints/cv/best_model_fold3.pt new file mode 100644 index 0000000000000000000000000000000000000000..dbb9ade9a38df92346d98e0cf1c87c23928796da --- /dev/null +++ b/checkpoints/cv/best_model_fold3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:937ba7d3640334ef41b05f15bc4bc2581ba60f8fe3711ff9fe2f406154db05f7 +size 43339 diff --git a/checkpoints/cv/best_model_fold4.pt b/checkpoints/cv/best_model_fold4.pt new file mode 100644 index 0000000000000000000000000000000000000000..b1625d72372e5cf089a923011ba0223c4b0ef561 --- /dev/null +++ b/checkpoints/cv/best_model_fold4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44f2863172d498f05bd4b34fb7fa56b131cc74e9cf2c986d78537acc85fdbaf6 +size 43339 diff --git a/checkpoints/multimodal_v2/feature_spec.json b/checkpoints/multimodal_v2/feature_spec.json new file mode 100644 index 0000000000000000000000000000000000000000..f7243337a314abb79b9b194c7ec954e37493c103 --- /dev/null +++ b/checkpoints/multimodal_v2/feature_spec.json @@ -0,0 +1,32 @@ +{ + "audio": { + "name": "audio", + "expected_dim": 62, + "temporal": true, + "preprocessor_class": null, + "extractor_class": null, + "pca_enabled": false, + "pca_target_dim": null, + "normalizer_type": "standard" + }, + "video": { + "name": "video", + "expected_dim": 49, + "temporal": true, + "preprocessor_class": null, + "extractor_class": null, + "pca_enabled": false, + "pca_target_dim": null, + "normalizer_type": "standard" + }, + "text": { + "name": "text", + "expected_dim": 384, + "temporal": true, + "preprocessor_class": null, + "extractor_class": null, + "pca_enabled": false, + "pca_target_dim": null, + "normalizer_type": "standard" + } +} \ No newline at end of file diff --git a/checkpoints/multimodal_v2/preprocessors/audio_preprocessor.pkl b/checkpoints/multimodal_v2/preprocessors/audio_preprocessor.pkl new file mode 100644 index 0000000000000000000000000000000000000000..922e76adc29afbe1a65bda5089f2468b0e3e7055 --- /dev/null +++ b/checkpoints/multimodal_v2/preprocessors/audio_preprocessor.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc6356119bc5f47417ee3f5fd57585a9c83e4258a83aa223c4ab34f73714b2af +size 11640 diff --git a/checkpoints/multimodal_v2/preprocessors/text_preprocessor.pkl b/checkpoints/multimodal_v2/preprocessors/text_preprocessor.pkl new file mode 100644 index 0000000000000000000000000000000000000000..fadabb5cfada7f4423e94a1b57f508d902794bfa --- /dev/null +++ b/checkpoints/multimodal_v2/preprocessors/text_preprocessor.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed802752135f85d0d847b33077e36492a27511ad35df1ca5cfcfa6b9a6af0226 +size 264428 diff --git a/checkpoints/multimodal_v2/preprocessors/video_preprocessor.pkl b/checkpoints/multimodal_v2/preprocessors/video_preprocessor.pkl new file mode 100644 index 0000000000000000000000000000000000000000..599e91f6787e770fb6e0df026d3de0099ae4af92 --- /dev/null +++ b/checkpoints/multimodal_v2/preprocessors/video_preprocessor.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06b856001f26ddfe1b74400f476d6c2b80e000b67bd571f29e4b8857cd129df4 +size 11016 diff --git a/checkpoints/multimodal_v3/feature_spec.json b/checkpoints/multimodal_v3/feature_spec.json new file mode 100644 index 0000000000000000000000000000000000000000..f7243337a314abb79b9b194c7ec954e37493c103 --- /dev/null +++ b/checkpoints/multimodal_v3/feature_spec.json @@ -0,0 +1,32 @@ +{ + "audio": { + "name": "audio", + "expected_dim": 62, + "temporal": true, + "preprocessor_class": null, + "extractor_class": null, + "pca_enabled": false, + "pca_target_dim": null, + "normalizer_type": "standard" + }, + "video": { + "name": "video", + "expected_dim": 49, + "temporal": true, + "preprocessor_class": null, + "extractor_class": null, + "pca_enabled": false, + "pca_target_dim": null, + "normalizer_type": "standard" + }, + "text": { + "name": "text", + "expected_dim": 384, + "temporal": true, + "preprocessor_class": null, + "extractor_class": null, + "pca_enabled": false, + "pca_target_dim": null, + "normalizer_type": "standard" + } +} \ No newline at end of file diff --git a/checkpoints/multimodal_v3/preprocessors/audio_preprocessor.pkl b/checkpoints/multimodal_v3/preprocessors/audio_preprocessor.pkl new file mode 100644 index 0000000000000000000000000000000000000000..45ff9ac69c0c51886c824e25ec053b9a394af3a5 --- /dev/null +++ b/checkpoints/multimodal_v3/preprocessors/audio_preprocessor.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caff65d8a1812be62c39d70901e1801791631910cfda40e08443a04ae8efa30d +size 11733 diff --git a/checkpoints/multimodal_v3/preprocessors/text_preprocessor.pkl b/checkpoints/multimodal_v3/preprocessors/text_preprocessor.pkl new file mode 100644 index 0000000000000000000000000000000000000000..5caa7d38561e129fced21765c319123b77f1f1c6 --- /dev/null +++ b/checkpoints/multimodal_v3/preprocessors/text_preprocessor.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ca025ec6356361869c37c00b5b5a4e5bf4b0307192801010889fc67ea495c8a +size 264521 diff --git a/checkpoints/multimodal_v3/preprocessors/video_preprocessor.pkl b/checkpoints/multimodal_v3/preprocessors/video_preprocessor.pkl new file mode 100644 index 0000000000000000000000000000000000000000..95605b4ec2be5a2697ddd9a3b02ce11e46075400 --- /dev/null +++ b/checkpoints/multimodal_v3/preprocessors/video_preprocessor.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:963825c9464b72a6986599744b34fad3303529b2040773cfc2470e675d0650dd +size 11109 diff --git a/checkpoints/multimodal_v4/feature_spec.json b/checkpoints/multimodal_v4/feature_spec.json new file mode 100644 index 0000000000000000000000000000000000000000..f7243337a314abb79b9b194c7ec954e37493c103 --- /dev/null +++ b/checkpoints/multimodal_v4/feature_spec.json @@ -0,0 +1,32 @@ +{ + "audio": { + "name": "audio", + "expected_dim": 62, + "temporal": true, + "preprocessor_class": null, + "extractor_class": null, + "pca_enabled": false, + "pca_target_dim": null, + "normalizer_type": "standard" + }, + "video": { + "name": "video", + "expected_dim": 49, + "temporal": true, + "preprocessor_class": null, + "extractor_class": null, + "pca_enabled": false, + "pca_target_dim": null, + "normalizer_type": "standard" + }, + "text": { + "name": "text", + "expected_dim": 384, + "temporal": true, + "preprocessor_class": null, + "extractor_class": null, + "pca_enabled": false, + "pca_target_dim": null, + "normalizer_type": "standard" + } +} \ No newline at end of file diff --git a/checkpoints/multimodal_v4/preprocessors/audio_preprocessor.pkl b/checkpoints/multimodal_v4/preprocessors/audio_preprocessor.pkl new file mode 100644 index 0000000000000000000000000000000000000000..45ff9ac69c0c51886c824e25ec053b9a394af3a5 --- /dev/null +++ b/checkpoints/multimodal_v4/preprocessors/audio_preprocessor.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caff65d8a1812be62c39d70901e1801791631910cfda40e08443a04ae8efa30d +size 11733 diff --git a/checkpoints/multimodal_v4/preprocessors/text_preprocessor.pkl b/checkpoints/multimodal_v4/preprocessors/text_preprocessor.pkl new file mode 100644 index 0000000000000000000000000000000000000000..5caa7d38561e129fced21765c319123b77f1f1c6 --- /dev/null +++ b/checkpoints/multimodal_v4/preprocessors/text_preprocessor.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ca025ec6356361869c37c00b5b5a4e5bf4b0307192801010889fc67ea495c8a +size 264521 diff --git a/checkpoints/multimodal_v4/preprocessors/video_preprocessor.pkl b/checkpoints/multimodal_v4/preprocessors/video_preprocessor.pkl new file mode 100644 index 0000000000000000000000000000000000000000..95605b4ec2be5a2697ddd9a3b02ce11e46075400 --- /dev/null +++ b/checkpoints/multimodal_v4/preprocessors/video_preprocessor.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:963825c9464b72a6986599744b34fad3303529b2040773cfc2470e675d0650dd +size 11109 diff --git a/checkpoints/pca_cnn_transform.pkl b/checkpoints/pca_cnn_transform.pkl new file mode 100644 index 0000000000000000000000000000000000000000..e4259baa5b7ac55731c4995715a5c94289d65877 --- /dev/null +++ b/checkpoints/pca_cnn_transform.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:654fe52593b0a178def46f0cb3479dd4558e157233b9a7f3ed031df0a98b131d +size 1059084 diff --git a/checkpoints/scalers/feature_scalers.pkl b/checkpoints/scalers/feature_scalers.pkl new file mode 100644 index 0000000000000000000000000000000000000000..977108854144230e77b19f6632e27d2f639644d3 --- /dev/null +++ b/checkpoints/scalers/feature_scalers.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:878445280c1c0c5d1799346f07d08d0e915d27ed4b1d2ed1663c99ca440250d1 +size 1942 diff --git a/checkpoints/scalers/pca_reducer.pkl b/checkpoints/scalers/pca_reducer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..6aa97f3b42a85bb6af048c3411d39e8481af908a --- /dev/null +++ b/checkpoints/scalers/pca_reducer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17451fb4ae35aeb65c907cc3dd78ca2663f8f57473675e55d1d253e1e2300227 +size 120065 diff --git a/configs/feature_extraction.yaml b/configs/feature_extraction.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01c3a5fa8475984161c53336d06a24dec89d6ed2 --- /dev/null +++ b/configs/feature_extraction.yaml @@ -0,0 +1,17 @@ +audio: + sample_rate: 16000 + frame_rate: 8 + chunk_duration: 2.0 + chunk_overlap: 0.5 + mfcc_n_mfcc: 13 + hubert_model_name: facebook/hubert-base-ls960 + feature_backends: + - mfcc + - hubert +video: + frame_rate: 8 + temporal_window: 32 + openface_csv_key: openface + embedding_backend: resnet50 +text: + transformer_model: sentence-transformers/all-mpnet-base-v2 diff --git a/configs/inference.yaml b/configs/inference.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6f64a0e0138dbdef6d5cf4ef12a24d38556c593 --- /dev/null +++ b/configs/inference.yaml @@ -0,0 +1,19 @@ +artifacts: + checkpoint_path: checkpoints/best_model.pt +model: + audio_dim: 33 + video_dim: 40 + text_dim: 163 + fusion_dim: 32 + num_questions: 8 + fusion_mode: hybrid + encoder_type: conv_gru + num_layers: 1 + dropout: 0.1 +inference: + device: auto + batch_size: 4 +api: + port: 8000 +logging: + log_dir: logs diff --git a/configs/training.yaml b/configs/training.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1155ea5767b069bcc75eb8c24ede18a92291c635 --- /dev/null +++ b/configs/training.yaml @@ -0,0 +1,55 @@ +dataset: + feature_dir: data/npz + id_column: participant_id + label_column: phq_score + train_split: data/splits/train_split.csv + val_split: data/splits/dev_split.csv +features: + audio_dim: 62 + text_dim: 384 + video_dim: 49 +logging: + experiment_name: multimodal_v4 + log_dir: logs_v4 +loss: + # Keep exact weights from multimodal_v3 that achieved CCC=0.5058 + ccc_weight: 30.0 + regression_weight: 0.3 + question_weight: 0.25 + classification_weight: 0.05 + confidence_weight: 0.01 + entropy_weight: 0.2 + # Zero gate_balance → text can freely exceed 1/3 without competing loss + gate_balance_weight: 0.0 + # Text gate preference — penalise when text gate < 0.40 + text_gate_weight: 0.25 + text_gate_target: 0.40 +model: + # Exact dims from multimodal_v3 (proven CCC=0.5058, 91K params) + audio_dim: 33 + video_dim: 40 + text_dim: 163 + dropout: 0.45 + encoder_type: conv_gru + fusion_dim: 32 + fusion_mode: hybrid + modality_dropout: 0.20 + num_heads: 4 + num_layers: 1 + num_questions: 8 +regularization: + feature_noise_std: 0.06 + temporal_dropout_rate: 0.20 +training: + batch_size: 16 + early_stopping_min_delta: 0.001 + early_stopping_patience: 200 + grad_clip_norm: 1.0 + learning_rate: 0.001 + lr_factor: 0.5 + lr_patience: 30 + max_epochs: 1000 + num_workers: 0 + use_amp: true + weight_decay: 8e-3 + mixup_alpha: 0.3 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..2b8f8c8f98457c2e0027aea4ac0904fd50cbccd0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,26 @@ +torch>=2.1.2 +torchaudio>=2.1.2 +torchvision>=0.16.2 +numpy>=1.24.4 +scikit-learn>=1.3.2 +pandas>=2.0.3 +librosa>=0.10.1 +soundfile>=0.12.1 +pydub>=0.25.1 +opensmile>=2.4.2 +transformers>=4.36.2,<5.6 +sentence-transformers>=2.2.2,<5.5 +fastapi>=0.104.1 +uvicorn>=0.24.0 +slowapi>=0.1.9 +python-dotenv>=1.0.0 +pyyaml>=6.0.1 +sqlalchemy>=2.0.23 +tqdm>=4.66.1 +joblib>=1.3.2 +pydantic>=2.5.3 +requests>=2.31.0 +python-multipart>=0.0.6 +scipy>=1.11.4 +h5py>=3.10.0 +matplotlib>=3.8.2 \ No newline at end of file diff --git a/scripts/__pycache__/train.cpython-312.pyc b/scripts/__pycache__/train.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..092a71d8379ddc4c10a0bc967b6a391747252db0 Binary files /dev/null and b/scripts/__pycache__/train.cpython-312.pyc differ diff --git a/scripts/__pycache__/train_v2.cpython-312.pyc b/scripts/__pycache__/train_v2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4b3422443475396c7675e3c6e6ec913f05808cdc Binary files /dev/null and b/scripts/__pycache__/train_v2.cpython-312.pyc differ diff --git a/scripts/auto_train.py b/scripts/auto_train.py new file mode 100644 index 0000000000000000000000000000000000000000..59e71eb143167843a399c161b5c3f30d8c2a6fea --- /dev/null +++ b/scripts/auto_train.py @@ -0,0 +1,51 @@ +import os +import subprocess +import yaml + +config_path = "configs/training.yaml" + +def run_training(): + print("Running training...") + result = subprocess.run(["python3", "scripts/train_v2.py", "--config", config_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + + best_ccc = -1.0 + for line in result.stdout.split('\n'): + if "--> New best model saved with CCC:" in line: + try: + ccc = float(line.split(":")[-1].strip()) + if ccc > best_ccc: + best_ccc = ccc + except: + pass + print(f"Training finished. Best CCC: {best_ccc}") + if best_ccc == -1.0: + print("Training failed or did not report CCC. Output:") + print(result.stdout) + return best_ccc + +def update_config(lr, dropout, ccc_weight): + with open(config_path, "r") as f: + config = yaml.safe_load(f) + + config["model"]["dropout"] = dropout + config["training"]["learning_rate"] = lr + config["loss"]["ccc_weight"] = ccc_weight + + with open(config_path, "w") as f: + yaml.safe_dump(config, f) + +lrs = [5e-4, 1e-4, 8e-4] +dropouts = [0.1, 0.2, 0.3] +ccc_weights = [1.5, 2.0, 3.0] + +for lr in lrs: + for drop in dropouts: + for cw in ccc_weights: + print(f"Testing LR: {lr}, Dropout: {drop}, CCC Weight: {cw}") + update_config(lr, drop, cw) + ccc = run_training() + if ccc >= 0.55: + print(f"Success! Reached {ccc} CCC with LR={lr}, Drop={drop}, CCC Weight={cw}") + exit(0) + +print("Failed to reach 0.55 CCC after all trials.") diff --git a/scripts/evaluate.py b/scripts/evaluate.py new file mode 100644 index 0000000000000000000000000000000000000000..74c766f3bf3fd9618c12f747abc122faad23489a --- /dev/null +++ b/scripts/evaluate.py @@ -0,0 +1,68 @@ +import argparse +import logging +import sys +from pathlib import Path + +ROOT_DIR = Path(__file__).resolve().parents[1] +if str(ROOT_DIR) not in sys.path: + sys.path.insert(0, str(ROOT_DIR)) + +import yaml +import torch +from torch.utils.data import DataLoader + +from src.dataset.builder import DatasetBuilder +from src.dataset.collate import multimodal_collate_fn +from src.models.multimodal_model import MultimodalDepressionModel +from src.training.trainer import Trainer +from src.utils.logging import configure_logging +from src.evaluation.evaluator import evaluate_predictions + + +def load_config(path: Path) -> dict: + with open(path, "r", encoding="utf-8") as handle: + return yaml.safe_load(handle) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Evaluate ModelV2 checkpoint") + parser.add_argument("--config", type=Path, default=Path("configs/inference.yaml")) + parser.add_argument("--checkpoint", type=Path, required=True) + args = parser.parse_args() + + config = load_config(args.config) + configure_logging(Path(config.get("logging", {}).get("log_dir", "logs"))) + logger = logging.getLogger(__name__) + + val_dataset = DatasetBuilder.build_from_csv( + split_csv_path=Path(config["dataset"]["val_split"]), + feature_dir=Path(config["dataset"]["feature_dir"]), + id_column=config["dataset"].get("id_column", "participant_id"), + label_column=config["dataset"].get("label_column", "phq_total"), + augment=False, + ) + val_loader = DataLoader(val_dataset, batch_size=config.get("inference", {}).get("batch_size", 4), shuffle=False, collate_fn=multimodal_collate_fn) + + model = MultimodalDepressionModel(**config["model"]) + model.load_state_dict(torch.load(args.checkpoint, map_location="cpu")) + model.eval() + + y_true, y_pred, q_true, q_pred, cls_true, cls_pred = [], [], [], [], [], [] + with torch.no_grad(): + for batch in val_loader: + output = model( + batch["audio"], batch["video"], batch["text"], batch["audio_mask"], batch["video_mask"], batch["text_mask"], + ) + y_true.extend(batch["phq_total"].tolist()) + y_pred.extend(output["phq_total"].tolist()) + q_true.extend(batch["phq_questions"].tolist()) + q_pred.extend(output["phq_questions"].tolist()) + cls_true.extend(batch["classification"].tolist()) + cls_pred.extend(torch.sigmoid(output["classification"]).tolist()) + + results = evaluate_predictions(y_true, y_pred, q_true, q_pred, cls_true, cls_pred) + logger.info(f"Evaluation results: {results}") + + +if __name__ == "__main__": + main() diff --git a/scripts/extract_features.py b/scripts/extract_features.py new file mode 100644 index 0000000000000000000000000000000000000000..8b38ad79a77b87f8b3f0aae5dfbb3791cfc2ca66 --- /dev/null +++ b/scripts/extract_features.py @@ -0,0 +1,54 @@ +import argparse +from pathlib import Path +from typing import Dict, Any + +import yaml + +from src.features.audio_features import AudioFeatureExtractor +from src.features.video_features import VideoFeatureExtractor +from src.features.text_features import TextFeatureExtractor +from src.features.feature_store import save_features +from src.preprocessing.audio_preprocessor import AudioPreprocessor +from src.preprocessing.text_preprocessor import TextPreprocessor + + +def load_config(path: Path) -> Dict[str, Any]: + with open(path, "r", encoding="utf-8") as handle: + return yaml.safe_load(handle) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Extract ModelV2 multimodal features") + parser.add_argument("--config", type=Path, default=Path("configs/feature_extraction.yaml")) + parser.add_argument("--input-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + + config = load_config(args.config) + audio_preprocessor = AudioPreprocessor(**config["audio"]) + audio_extractor = AudioFeatureExtractor(config["audio"]) + video_extractor = VideoFeatureExtractor(config["video"]) + text_preprocessor = TextPreprocessor() + text_extractor = TextFeatureExtractor(config["text"]["transformer_model"]) + + for subject_dir in args.input_dir.iterdir(): + if not subject_dir.is_dir(): + continue + participant_id = subject_dir.name + audio_path = subject_dir / "audio.wav" + transcript_path = subject_dir / "transcript.txt" + openface_path = subject_dir / "openface.csv" + if not audio_path.exists() or not transcript_path.exists() or not openface_path.exists(): + continue + waveform = audio_preprocessor.load_audio(audio_path) + audio_chunks = audio_preprocessor.preprocess(waveform) + audio_features = {"audio": audio_extractor.extract_modern(waveform)} + video_behavior = video_extractor.extract_openface(openface_path)["behavior"] + text = transcript_path.read_text(encoding="utf-8") + chunks = text_preprocessor.preprocess(text) + text_features = {"text": text_extractor.encode(chunks)} + save_features(args.output_dir / f"{participant_id}.npz", { + **audio_features, + "video": video_behavior, + **text_features, + }) diff --git a/scripts/infer_live_v2.py b/scripts/infer_live_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..040e57f89a36cc0cc5cb47b49c079b057bf3013e --- /dev/null +++ b/scripts/infer_live_v2.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Example live inference script for multimodal depression detection. + +Demonstrates how to use the LiveInferencePipeline for real-time predictions +from webcam/video/audio/transcript inputs. +""" + +import argparse +import sys +from pathlib import Path + +ROOT_DIR = Path(__file__).resolve().parents[2] +if str(ROOT_DIR) not in sys.path: + sys.path.insert(0, str(ROOT_DIR)) + +import torch +from src.inference.live_pipeline import LiveInferencePipeline +from src.features.specs import FeatureSpecification +from src.inference.validation import FeatureCheckpoint + + +def main() -> None: + """Example usage of live inference pipeline.""" + parser = argparse.ArgumentParser(description="Live inference example") + parser.add_argument("--checkpoint", type=Path, required=True, help="Model checkpoint path") + parser.add_argument("--experiment", type=str, required=True, help="Experiment name for loading preprocessors") + parser.add_argument("--audio", type=Path, help="Path to audio file") + parser.add_argument("--openface-csv", type=Path, help="Path to OpenFace CSV (video features)") + parser.add_argument("--transcript", type=str, help="Transcript text") + parser.add_argument("--device", type=str, default="auto", help="Device (auto/cuda/cpu)") + parser.add_argument("--verbose", action="store_true", help="Verbose output") + args = parser.parse_args() + + # Load feature spec and preprocessors + checkpoint_manager = FeatureCheckpoint(Path("checkpoints")) + + try: + feature_spec = checkpoint_manager.load_feature_spec(args.experiment) + preprocessors = checkpoint_manager.load_preprocessors(args.experiment) + except FileNotFoundError as e: + print(f"Error: {e}") + print(f"Make sure to run training script with --experiment-name {args.experiment}") + sys.exit(1) + + # Model config (same as training) + model_config = { + "audio_dim": feature_spec.audio_dim, + "video_dim": feature_spec.video_dim, + "text_dim": feature_spec.text_dim, + "fusion_dim": 256, + "num_questions": 8, + "encoder_type": "transformer", + "num_heads": 4, + "num_layers": 2, + "dropout": 0.2, + "fusion_mode": "hybrid", + } + + # Initialize pipeline + pipeline = LiveInferencePipeline( + checkpoint_path=args.checkpoint, + model_config=model_config, + feature_spec=feature_spec, + preprocessors=preprocessors, + device=args.device, + ) + + if args.verbose: + print(f"Model loaded from: {args.checkpoint}") + print(f"Feature spec: audio={feature_spec.audio_dim}D, video={feature_spec.video_dim}D, text={feature_spec.text_dim}D") + + # Run inference + try: + result = pipeline.predict_single( + audio_path=args.audio, + video_openface_csv=args.openface_csv, + transcript=args.transcript, + verbose=args.verbose, + ) + + # Print results + print(pipeline.predict_and_explain( + audio_path=args.audio, + video_openface_csv=args.openface_csv, + transcript=args.transcript, + )) + + except Exception as e: + print(f"Inference error: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/serve.py b/scripts/serve.py new file mode 100644 index 0000000000000000000000000000000000000000..063d4ad27e393c1c353fa78dcf703fa73ffcad6c --- /dev/null +++ b/scripts/serve.py @@ -0,0 +1,37 @@ +import argparse +import uvicorn +import sys +from pathlib import Path + +ROOT_DIR = Path(__file__).resolve().parents[1] +if str(ROOT_DIR) not in sys.path: + sys.path.insert(0, str(ROOT_DIR)) + +from src.api.app import create_app +from src.utils.config import load_config + + +def main() -> None: + parser = argparse.ArgumentParser(description="Serve ModelV2 inference API") + parser.add_argument("--config", type=Path, default=Path("configs/inference.yaml")) + parser.add_argument("--host", type=str, default="0.0.0.0") + parser.add_argument("--port", type=int, default=None) + parser.add_argument("--log-level", type=str, default="info") + parser.add_argument("--reload", action="store_true") + args = parser.parse_args() + config = load_config(args.config) + if args.port is not None: + config["api"]["port"] = args.port + checkpoint_path = config["artifacts"]["checkpoint_path"] + model_config = config["model"] + device = config.get("inference", {}).get("device", "auto") + app = create_app(str(checkpoint_path), model_config, device=device) + + # We must pass the import string 'src.api.app:create_app' if we want reload to work, + # but since app is already created, reload might not work properly without restructuring. + # We'll just pass the app directly. + uvicorn.run(app, host=args.host, port=config.get("api", {}).get("port", 8000), log_level=args.log_level.lower()) + + +if __name__ == "__main__": + main() diff --git a/scripts/smoke_infer_test.py b/scripts/smoke_infer_test.py new file mode 100644 index 0000000000000000000000000000000000000000..10bc1c42c3cda0d1f4035d0f743b52807db2752e --- /dev/null +++ b/scripts/smoke_infer_test.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Smoke test: load inference config and instantiate ModelV2Inferencer using CPU.""" +import sys +from pathlib import Path +import traceback + +from src.utils.config import load_config + + +def main(): + try: + repo_root = Path(__file__).resolve().parents[1] + cfg_path = repo_root / "configs" / "inference.yaml" + cfg = load_config(cfg_path) + # Force CPU to avoid CUDA/device autodetection issues + cfg.setdefault("inference", {})["device"] = "cpu" + + checkpoint = Path(cfg["artifacts"]["checkpoint_path"]) if "artifacts" in cfg else Path("checkpoints/best_model.pt") + if not checkpoint.is_absolute(): + checkpoint = (repo_root / checkpoint).resolve() + + print("Config loaded. Checkpoint:", checkpoint) + print("Exists:", checkpoint.exists()) + + from src.inference.inferencer import ModelV2Inferencer + + model_cfg = cfg.get("model", {}) + infer = ModelV2Inferencer(checkpoint, model_cfg, device="cpu") + print("Model instantiated on device:", infer.device) + print("Smoke test OK") + except Exception: + traceback.print_exc() + sys.exit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/train.py b/scripts/train.py new file mode 100644 index 0000000000000000000000000000000000000000..d00419c8b65b6ff1f1842f8a146405106f9ab316 --- /dev/null +++ b/scripts/train.py @@ -0,0 +1,93 @@ +import argparse +import logging +import sys +from pathlib import Path + +ROOT_DIR = Path(__file__).resolve().parents[1] +if str(ROOT_DIR) not in sys.path: + sys.path.insert(0, str(ROOT_DIR)) + +import yaml +import torch +from torch.utils.data import DataLoader + +from src.dataset.builder import DatasetBuilder +from src.dataset.collate import multimodal_collate_fn +from src.models.multimodal_model import MultimodalDepressionModel +from src.training.trainer import Trainer +from src.utils.logging import configure_logging + + +def load_config(path: Path) -> dict: + with open(path, "r", encoding="utf-8") as handle: + return yaml.safe_load(handle) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Train ModelV2 multimodal depression model") + parser.add_argument("--config", type=Path, default=Path("configs/training.yaml")) + args = parser.parse_args() + + config = load_config(args.config) + run_dir = configure_logging(Path(config.get("logging", {}).get("log_dir", "logs")), experiment_name=config.get("logging", {}).get("experiment_name", "trimodal")) + logger = logging.getLogger(__name__) + + train_dataset = DatasetBuilder.build_from_csv( + split_csv_path=Path(config["dataset"]["train_split"]), + feature_dir=Path(config["dataset"]["feature_dir"]), + id_column=config["dataset"].get("id_column", "participant_id"), + label_column=config["dataset"].get("label_column", "phq_total"), + augment=True, + ) + val_dataset = DatasetBuilder.build_from_csv( + split_csv_path=Path(config["dataset"]["val_split"]), + feature_dir=Path(config["dataset"]["feature_dir"]), + id_column=config["dataset"].get("id_column", "participant_id"), + label_column=config["dataset"].get("label_column", "phq_total"), + augment=False, + ) + + train_loader = DataLoader( + train_dataset, + batch_size=int(config["training"].get("batch_size", 8)), + shuffle=True, + num_workers=int(config["training"].get("num_workers", 2)), + collate_fn=multimodal_collate_fn, + ) + val_loader = DataLoader( + val_dataset, + batch_size=int(config["training"].get("batch_size", 8)), + shuffle=False, + num_workers=int(config["training"].get("num_workers", 2)), + collate_fn=multimodal_collate_fn, + ) + + model_config = config.get("model", {}) + model = MultimodalDepressionModel(**model_config) + optimizer = torch.optim.AdamW( + model.parameters(), + lr=float(config["training"].get("learning_rate", 3e-4)), + weight_decay=float(config["training"].get("weight_decay", 1e-4)), + ) + scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( + optimizer, + mode="min", + factor=float(config["training"].get("lr_factor", 0.5)), + patience=int(config["training"].get("lr_patience", 3)), + ) + + trainer = Trainer( + model=model, + train_loader=train_loader, + val_loader=val_loader, + optimizer=optimizer, + scheduler=scheduler, + config=config["training"], + ) + logger.info("Starting ModelV2 training") + history = trainer.train() + logger.info(f"Training complete: {history}") + + +if __name__ == "__main__": + main() diff --git a/scripts/train_v2.py b/scripts/train_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..34e526c9327921d3edcaccdc22cb1479e2dfebea --- /dev/null +++ b/scripts/train_v2.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +"""Improved training script for multimodal depression detection. + +Features: +- Consistent feature pipeline with validation +- Advanced regularization (EMA, checkpoint averaging, curriculum learning) +- Comprehensive logging and checkpointing +- Small dataset optimization +""" + +import argparse +import logging +import sys +from pathlib import Path +import numpy as np + +def convert_numpy(obj): + if isinstance(obj, np.integer): + return int(obj) + if isinstance(obj, np.floating): + return float(obj) + if isinstance(obj, np.ndarray): + return obj.tolist() + if isinstance(obj, dict): + return {k: convert_numpy(v) for k, v in obj.items()} + if isinstance(obj, list): + return [convert_numpy(i) for i in obj] + return obj + +ROOT_DIR = Path(__file__).resolve().parents[1] +if str(ROOT_DIR) not in sys.path: + sys.path.insert(0, str(ROOT_DIR)) + +import yaml +import torch +from torch.utils.data import DataLoader + +from src.dataset.builder import DatasetBuilder +from src.dataset.collate import multimodal_collate_fn +from src.dataset.multimodal_dataset import MultimodalSample +from src.models.multimodal_model import MultimodalDepressionModel +from src.training.trainer import Trainer +from src.training.optimization import EMA, WarmupScheduler, CurriculumLearning +from src.features.specs import FeatureSpecification +from src.features.preprocessing import FeaturePreprocessor +from src.inference.validation import TrainInferenceConsistencyChecker, FeatureCheckpoint +from src.utils.logging import configure_logging + + +def load_config(path: Path) -> dict: + """Load YAML configuration.""" + with open(path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + + +def setup_feature_specs(config: dict) -> FeatureSpecification: + """Set up feature specifications from config.""" + # Use defaults or override from config + feature_config = config.get("features", {}) + + # Create feature specs matching the config + from src.features.specs import ModalitySpec + + # Determine which specs to use based on dimensions + audio_dim = feature_config.get("audio_dim", 39) + video_dim = feature_config.get("video_dim", 38) + text_dim = feature_config.get("text_dim", 384) + + # For now, use standard specs (can be enhanced to support other combinations) + spec = FeatureSpecification( + audio_spec=ModalitySpec(name="audio", expected_dim=audio_dim, temporal=True), + video_spec=ModalitySpec(name="video", expected_dim=video_dim, temporal=True), + text_spec=ModalitySpec(name="text", expected_dim=text_dim, temporal=True), + ) + + # Removed strict assertions because PCA determines final dimensions dynamically + return spec + + +def setup_preprocessors(config: dict, feature_spec: FeatureSpecification) -> dict: + """Set up feature preprocessors (normalization, PCA).""" + preprocessors = {} + + # Audio preprocessor + audio_prep = FeaturePreprocessor( + normalizer_type="standard", + apply_pca=True, + pca_components=0.80, + pca_whiten=True, + ) + preprocessors["audio"] = audio_prep + + # Video preprocessor + video_prep = FeaturePreprocessor( + normalizer_type="standard", + apply_pca=True, + pca_components=0.80, + pca_whiten=True, + ) + preprocessors["video"] = video_prep + + # Text preprocessor (Aggressive reduction) + text_prep = FeaturePreprocessor( + normalizer_type="standard", + apply_pca=True, + pca_components=0.80, + pca_whiten=True, + ) + preprocessors["text"] = text_prep + + return preprocessors + + +def _try_load_preprocessors(checkpoint_dir: Path, logger: logging.Logger) -> dict | None: + """Try to load already-saved PCA preprocessors from the most recent checkpoint. + + Reuses the PCA transform computed in a previous training run so that: + - dimensions stay consistent with the saved best_model.pt + - no refitting noise between runs + - faster startup + + Returns None if no valid checkpoint exists. + """ + # Preference order: multimodal_v2 (best validated run), then any other + candidates = ["multimodal_v3", "multimodal_v2", "multimodal_v1"] + for name in candidates: + prep_dir = checkpoint_dir / name / "preprocessors" + if not prep_dir.exists(): + continue + missing = [m for m in ["audio", "video", "text"] if not (prep_dir / f"{m}_preprocessor.pkl").exists()] + if missing: + continue + try: + from src.inference.validation import FeatureCheckpoint + manager = FeatureCheckpoint(checkpoint_dir) + preps = manager.load_preprocessors(name) + if all(preps.get(m) and preps[m].output_dim for m in ["audio", "video", "text"]): + logger.info( + f"Loaded existing PCA preprocessors from '{name}' checkpoint: " + f"audio={preps['audio'].output_dim}d " + f"video={preps['video'].output_dim}d " + f"text={preps['text'].output_dim}d" + ) + return preps + except Exception as exc: + logger.warning(f"Could not load preprocessors from '{name}': {exc}") + return None + + +def create_datasets_with_preprocessing( + config: dict, + preprocessors: dict, + feature_spec: FeatureSpecification, + logger: logging.Logger, + checkpoint_dir: Path | None = None, +) -> tuple: + """Create training and validation datasets with preprocessing. + + Tries to load already-saved PCA preprocessors first. Only refits if none exist. + """ + + reg_cfg = config.get("regularization", {}) + feat_noise = float(reg_cfg.get("feature_noise_std", 0.01)) + temp_drop = float(reg_cfg.get("temporal_dropout_rate", 0.10)) + + train_dataset_raw = DatasetBuilder.build_from_csv( + split_csv_path=Path(config["dataset"]["train_split"]), + feature_dir=Path(config["dataset"]["feature_dir"]), + id_column=config["dataset"].get("id_column", "participant_id"), + label_column=config["dataset"].get("label_column", "phq_total"), + augment=True, + temporal_dropout_rate=temp_drop, + feature_noise_std=feat_noise, + ) + + val_dataset_raw = DatasetBuilder.build_from_csv( + split_csv_path=Path(config["dataset"]["val_split"]), + feature_dir=Path(config["dataset"]["feature_dir"]), + id_column=config["dataset"].get("id_column", "participant_id"), + label_column=config["dataset"].get("label_column", "phq_total"), + augment=False, + temporal_dropout_rate=0.0, + feature_noise_std=0.0, + ) + + logger.info(f"Loaded {len(train_dataset_raw)} training samples") + logger.info(f"Loaded {len(val_dataset_raw)} validation samples") + + # Try to reuse already-saved PCA preprocessors (avoids refitting noise) + saved = _try_load_preprocessors(checkpoint_dir or Path("checkpoints"), logger) + if saved: + preprocessors = saved + logger.info("Using saved PCA preprocessors — skipping refit.") + else: + # Fit PCA preprocessors fresh on training data + logger.info("No saved preprocessors found. Fitting PCA from scratch...") + for modality in ["audio", "video", "text"]: + features_list = [ + s.get(modality) for s in train_dataset_raw.samples + if s.get(modality) is not None + ] + if features_list: + all_features = np.vstack(features_list) + preprocessors[modality].fit(all_features) + logger.info( + f" {modality}: PCA fitted → {preprocessors[modality].output_dim}d " + f"(from {all_features.shape[1]}d, {all_features.shape[0]} rows)" + ) + + # Apply preprocessors to both splits + logger.info("Applying PCA preprocessors to datasets...") + for split_name, dataset in [("train", train_dataset_raw), ("val", val_dataset_raw)]: + for sample in dataset.samples: + for modality in ["audio", "video", "text"]: + if sample.get(modality) is not None: + sample[modality] = preprocessors[modality].transform(sample[modality]) + logger.info( + f" {split_name}: applied PCA — audio now {dataset.samples[0]['audio'].shape[1] if dataset.samples[0].get('audio') is not None else 'N/A'}d" + ) + + return train_dataset_raw, val_dataset_raw, preprocessors + + +def main() -> None: + """Main training function.""" + parser = argparse.ArgumentParser(description="Train improved multimodal depression model") + parser.add_argument("--config", type=Path, default=Path("configs/training.yaml")) + parser.add_argument("--experiment-name", type=str, default="multimodal_v2") + parser.add_argument("--use-ema", action="store_true", help="Use exponential moving average") + parser.add_argument("--seed", type=int, default=42, help="Random seed for reproducibility") + args = parser.parse_args() + + # Fix random seed for reproducibility + import random + random.seed(args.seed) + np.random.seed(args.seed) + torch.manual_seed(args.seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(args.seed) + + # Load config + config = load_config(args.config) + + # Set up logging + run_dir = configure_logging( + Path(config.get("logging", {}).get("log_dir", "logs")), + experiment_name=args.experiment_name, + ) + logger = logging.getLogger(__name__) + + logger.info(f"Starting training: {args.experiment_name}") + logger.info(f"Config: {args.config}") + + # Feature specification and preprocessing + logger.info("Setting up feature pipeline...") + feature_spec = setup_feature_specs(config) + logger.info(f"Feature spec - Audio: {feature_spec.audio_dim}D, Video: {feature_spec.video_dim}D, Text: {feature_spec.text_dim}D") + + preprocessors = setup_preprocessors(config, feature_spec) + + checkpoint_dir = Path(config["training"].get("checkpoint_dir", "checkpoints")) + + # Create datasets — loads existing PCA preprocessors if available + train_dataset, val_dataset, preprocessors = create_datasets_with_preprocessing( + config, + preprocessors, + feature_spec, + logger, + checkpoint_dir=checkpoint_dir, + ) + + # Update model config dynamically based on PCA output dims + if preprocessors["audio"].output_dim: + config["model"]["audio_dim"] = preprocessors["audio"].output_dim + if preprocessors["video"].output_dim: + config["model"]["video_dim"] = preprocessors["video"].output_dim + if preprocessors["text"].output_dim: + config["model"]["text_dim"] = preprocessors["text"].output_dim + + logger.info(f"Final model dims - Audio: {config['model']['audio_dim']}, Video: {config['model']['video_dim']}, Text: {config['model']['text_dim']}") + + # Create dataloaders + train_loader = DataLoader( + train_dataset, + batch_size=int(config["training"].get("batch_size", 8)), + shuffle=True, + num_workers=int(config["training"].get("num_workers", 2)), + collate_fn=multimodal_collate_fn, + ) + val_loader = DataLoader( + val_dataset, + batch_size=int(config["training"].get("batch_size", 8)), + shuffle=False, + num_workers=int(config["training"].get("num_workers", 2)), + collate_fn=multimodal_collate_fn, + ) + + # Model + model_config = config.get("model", {}) + model = MultimodalDepressionModel(**model_config) + logger.info(f"Model parameters: {sum(p.numel() for p in model.parameters()):,}") + + # Optimizer and scheduler + optimizer = torch.optim.AdamW( + model.parameters(), + lr=float(config["training"].get("learning_rate", 1e-4)), + weight_decay=float(config["training"].get("weight_decay", 1e-4)), + ) + + # Warmup scheduler + warmup_scheduler = WarmupScheduler( + optimizer, + warmup_epochs=5, + total_epochs=int(config["training"].get("max_epochs", 100)), + base_lr=float(config["training"].get("learning_rate", 1e-4)), + ) + + # Reduce on plateau scheduler + plateau_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( + optimizer, + mode="min", + factor=float(config["training"].get("lr_factor", 0.5)), + patience=int(config["training"].get("lr_patience", 5)), + ) + + # EMA if enabled + ema = None + if args.use_ema: + ema = EMA(model, decay=0.999) + logger.info("Using exponential moving average") + + # Merge loss weights into training config for Trainer + training_config = config["training"].copy() + if "loss" in config: + training_config.update(config["loss"]) + + # Trainer + trainer = Trainer( + model=model, + train_loader=train_loader, + val_loader=val_loader, + optimizer=optimizer, + scheduler=plateau_scheduler, + config=training_config, + ) + + # Training + logger.info("Starting training...") + history = trainer.train() + + # Save best model info + best_loss = getattr(trainer, "best_loss", None) + best_ccc = getattr(trainer, "best_ccc", None) + logger.info("Training complete!") + logger.info(f"Best CCC: {best_ccc:.4f}" if best_ccc else "Best CCC: N/A") + + # Write result.json + result_data = { + "best_ccc": float(best_ccc) if best_ccc else 0.0, + "best_loss": float(best_loss) if best_loss else 0.0, + "experiment_name": args.experiment_name, + "config": convert_numpy(config), + "run_dir": str(run_dir), + } + import json + with open("result.json", "w") as f: + json.dump(result_data, f, indent=4) + logger.info(f"Best loss: {history.get('best_loss', 'N/A')}") + + # Save preprocessors for inference + checkpoint_manager = FeatureCheckpoint(Path(config["training"].get("checkpoint_dir", "checkpoints"))) + checkpoint_manager.save_preprocessors(preprocessors, args.experiment_name) + checkpoint_manager.save_feature_spec(feature_spec, args.experiment_name) + logger.info(f"Saved preprocessors and feature spec to {args.experiment_name}") + + +if __name__ == "__main__": + main() diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/__pycache__/__init__.cpython-312.pyc b/src/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aa2b73371b7954c06175e6f8849eff261e44f61d Binary files /dev/null and b/src/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/api/__init__.py b/src/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/api/__pycache__/__init__.cpython-312.pyc b/src/api/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..057230c1f812d09d289b68493ee2273b289a7dae Binary files /dev/null and b/src/api/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/api/__pycache__/app.cpython-312.pyc b/src/api/__pycache__/app.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57a7c05cedb12593b76f47d3a2f5fca209c72aa2 Binary files /dev/null and b/src/api/__pycache__/app.cpython-312.pyc differ diff --git a/src/api/__pycache__/routes.cpython-312.pyc b/src/api/__pycache__/routes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e8805b40ad4e9780e78b394cd990ee50791d21f0 Binary files /dev/null and b/src/api/__pycache__/routes.cpython-312.pyc differ diff --git a/src/api/__pycache__/schemas.cpython-312.pyc b/src/api/__pycache__/schemas.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2197d978ded5a4eb0076c7ca376b52cc2f7d409b Binary files /dev/null and b/src/api/__pycache__/schemas.cpython-312.pyc differ diff --git a/src/api/app.py b/src/api/app.py new file mode 100644 index 0000000000000000000000000000000000000000..b9bc983ad47ae4bb74f2e6f2dc7308d20b62f797 --- /dev/null +++ b/src/api/app.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import logging +import pickle +from pathlib import Path +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.gzip import GZipMiddleware +from contextlib import asynccontextmanager + +from src.api.routes import router, register_inferencer, get_inferencer +from src.features.text_features import TextFeatureExtractor +from src.inference.inferencer import ModelV2Inferencer +from src.utils.logging import configure_logging + +logger = logging.getLogger(__name__) + + +async def _warmup_model() -> None: + """Run a dummy forward pass so PyTorch JIT compiles on startup.""" + inferencer = get_inferencer() + if inferencer is None: + return + try: + dummy = { + "audio": [[0.0] * 33], + "audio_mask": [True], + } + inferencer.predict_single(dummy) + logger.info("Model warmup completed successfully.") + except Exception as exc: + logger.warning("Model warmup failed (non-fatal): %s", exc) + + +@asynccontextmanager +async def _lifespan(app: FastAPI): + await _warmup_model() + yield + + +def _load_preprocessors(checkpoint_path: str) -> dict: + """Load normalizer+PCA preprocessors for each modality from the training checkpoint.""" + checkpoint_dir = Path(checkpoint_path).parent + # Search in multimodal_v3 first, then generic preprocessors dir + candidates = [ + checkpoint_dir / "multimodal_v4" / "preprocessors", + checkpoint_dir / "multimodal_v3" / "preprocessors", + checkpoint_dir / "preprocessors", + ] + preproc_dir = next((d for d in candidates if d.exists()), None) + if preproc_dir is None: + logger.warning("No preprocessors directory found under %s", checkpoint_dir) + return {} + preprocessors = {} + for modality in ("audio", "video", "text"): + pkl_path = preproc_dir / f"{modality}_preprocessor.pkl" + if pkl_path.exists(): + try: + with open(pkl_path, "rb") as f: + preprocessors[modality] = pickle.load(f) + logger.info("Loaded %s preprocessor from %s", modality, pkl_path) + except Exception as exc: + logger.warning("Failed to load %s preprocessor: %s", modality, exc) + return preprocessors + + +def create_app(checkpoint_path: str, model_config: dict, device: str = "auto", text_model_name: str = "sentence-transformers/all-mpnet-base-v2") -> FastAPI: + configure_logging(Path("logs"), level="INFO") + inferencer = ModelV2Inferencer(Path(checkpoint_path), model_config, device=device) + # Attempt to load text PCA/reducer saved during training. Prefer explicit + # PCA reducer file if present, otherwise fall back to the generic scalers. + scalers_dir = Path(checkpoint_path).parent / "scalers" + pca_path = scalers_dir / "pca_reducer.pkl" + feature_scalers_path = scalers_dir / "feature_scalers.pkl" + preproc_path = pca_path if pca_path.exists() else (feature_scalers_path if feature_scalers_path.exists() else None) + text_extractor = TextFeatureExtractor(text_model_name, preprocessor_path=preproc_path) + preprocessors = _load_preprocessors(checkpoint_path) + register_inferencer(inferencer, text_extractor=text_extractor, preprocessors=preprocessors) + + app = FastAPI( + title="ModelV2 Depression Assessment API", + description="Production-grade FastAPI inference for multimodal depression assessment.", + version="2.0.0", + lifespan=_lifespan, + ) + app.add_middleware(GZipMiddleware, minimum_size=1024) + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], + ) + app.include_router(router, prefix="") + return app diff --git a/src/api/routes.py b/src/api/routes.py new file mode 100644 index 0000000000000000000000000000000000000000..b4875568460963f8c81bacea58ac633fc7dff203 --- /dev/null +++ b/src/api/routes.py @@ -0,0 +1,297 @@ +from typing import Optional, Dict, Any +import hashlib +import io +import json +import time +import logging +import numpy as np +from fastapi import APIRouter, File, Form, HTTPException, BackgroundTasks, UploadFile +from src.api.schemas import AudioPayload, VideoPayload, TextPayload, MultimodalPayload, PredictionResponse, HealthResponse +from src.features.text_features import TextFeatureExtractor +from src.inference.inferencer import ModelV2Inferencer + +router = APIRouter() +_inferencer: Optional[ModelV2Inferencer] = None +_text_extractor: Optional[TextFeatureExtractor] = None +_preprocessors: Dict[str, Any] = {} + +logger = logging.getLogger(__name__) + +# ── Route-level prediction cache (TTL = 1 hour, max 200 entries) ────────────── +_pred_cache: dict = {} +_PRED_CACHE_TTL = 3600 +_PRED_CACHE_MAX = 200 + + +def _pred_cache_key(payload_dict: dict) -> str: + try: + raw = json.dumps(payload_dict, sort_keys=True, default=str) + return hashlib.sha256(raw.encode()).hexdigest() + except Exception: + return "" + + +def _pred_cache_get(key: str) -> Optional[dict]: + entry = _pred_cache.get(key) + if entry and (time.monotonic() - entry["ts"]) < _PRED_CACHE_TTL: + return entry["v"] + return None + + +def _pred_cache_set(key: str, value: dict) -> None: + _pred_cache[key] = {"v": value, "ts": time.monotonic()} + if len(_pred_cache) > _PRED_CACHE_MAX: + oldest = sorted(_pred_cache.items(), key=lambda x: x[1]["ts"]) + for k, _ in oldest[: len(_pred_cache) - _PRED_CACHE_MAX]: + _pred_cache.pop(k, None) + + +def register_inferencer( + inferencer: ModelV2Inferencer, + text_extractor: Optional[TextFeatureExtractor] = None, + preprocessors: Optional[Dict[str, Any]] = None, +) -> None: + global _inferencer, _text_extractor, _preprocessors + _inferencer = inferencer + _text_extractor = text_extractor + if preprocessors: + _preprocessors = preprocessors + logger.info("Registered preprocessors for modalities: %s", list(preprocessors.keys())) + + +def _apply_preprocessor(features_list: Optional[list], modality: str) -> Optional[list]: + """Apply normalizer + PCA to raw features using the training preprocessors.""" + if not features_list or modality not in _preprocessors: + return features_list + preproc = _preprocessors[modality] + try: + arr = np.array(features_list, dtype=np.float32) + if arr.ndim == 1: + arr = arr.reshape(1, -1) + normalizer = preproc.get("normalizer") if isinstance(preproc, dict) else getattr(preproc, "normalizer", None) + pca = preproc.get("pca") if isinstance(preproc, dict) else getattr(preproc, "pca", None) + if normalizer is not None: + arr = normalizer.transform(arr) + if pca is not None: + arr = pca.transform(arr) + return arr.tolist() + except Exception as exc: + logger.warning("Preprocessing failed for %s (%s); passing features as-is", modality, exc) + return features_list + + +def get_inferencer() -> Optional[ModelV2Inferencer]: + return _inferencer + + +@router.get("/health", response_model=HealthResponse) +async def health() -> HealthResponse: + return HealthResponse( + status="ok", + model_loaded=_inferencer is not None, + description="ModelV2 inference service is available." if _inferencer else "Model is not loaded yet.", + ) + + +@router.post("/predict/audio", response_model=PredictionResponse) +async def predict_audio(payload: AudioPayload, background_tasks: BackgroundTasks) -> PredictionResponse: + if _inferencer is None: + raise HTTPException(status_code=503, detail="Inference model is not initialized.") + if not payload.audio_features: + raise HTTPException(status_code=400, detail="Audio features are required.") + start = time.perf_counter() + sample = { + "audio": payload.audio_features, + "audio_mask": [True] * len(payload.audio_features), + } + result = _inferencer.predict_single(sample) + duration = time.perf_counter() - start + result.setdefault("metadata", {}) + result["processing_details"] = {"total_seconds": duration, "stages": {"inference": duration}} + logger.info("predict_audio finished: duration=%.3fs classification=%.4f confidence=%.4f", duration, result.get("classification"), result.get("confidence")) + return PredictionResponse(**result) + + +@router.post("/predict/video", response_model=PredictionResponse) +async def predict_video(payload: VideoPayload) -> PredictionResponse: + if _inferencer is None: + raise HTTPException(status_code=503, detail="Inference model is not initialized.") + if not payload.video_features: + raise HTTPException(status_code=400, detail="Video features are required.") + start = time.perf_counter() + sample = { + "video": payload.video_features, + "video_mask": [True] * len(payload.video_features), + } + result = _inferencer.predict_single(sample) + duration = time.perf_counter() - start + result.setdefault("metadata", {}) + result["processing_details"] = {"total_seconds": duration, "stages": {"inference": duration}} + logger.info("predict_video finished: duration=%.3fs classification=%.4f confidence=%.4f", duration, result.get("classification"), result.get("confidence")) + return PredictionResponse(**result) + + +@router.post("/predict/text", response_model=PredictionResponse) +async def predict_text(payload: TextPayload) -> PredictionResponse: + if _inferencer is None: + raise HTTPException(status_code=503, detail="Inference model is not initialized.") + if payload.transcript is None and not payload.chunked_transcript: + raise HTTPException(status_code=400, detail="Transcript or chunked_transcript is required.") + if _text_extractor is None: + raise HTTPException(status_code=503, detail="Text feature extractor is not configured.") + chunks = payload.chunked_transcript or [payload.transcript] + start_total = time.perf_counter() + # measure text feature extraction + t0 = time.perf_counter() + features = _text_extractor.encode(chunks) + t_text = time.perf_counter() - t0 + # Validate feature dimensionality matches model expectation + expected_dim = _inferencer.model.text_input_dim if _inferencer else None + if expected_dim is not None: + if features.ndim == 1: + feat_dim = features.shape[0] + else: + feat_dim = features.shape[-1] + if feat_dim != expected_dim: + raise HTTPException(status_code=400, detail=f"Text embedding dim ({feat_dim}) does not match model expected dim ({expected_dim}). Ensure the saved text preprocessor (PCA/normalizer) is available in checkpoints/scalers.") + + sample = { + "text": features.tolist(), + "text_mask": [True] * len(chunks), + } + # measure model inference + t1 = time.perf_counter() + result = _inferencer.predict_single(sample) + t_infer = time.perf_counter() - t1 + + total = time.perf_counter() - start_total + result.setdefault("metadata", {}) + result["processing_details"] = {"total_seconds": total, "stages": {"text_extraction": t_text, "inference": t_infer}} + logger.info("predict_text finished: total=%.3fs text=%.3fs inference=%.3fs classification=%.4f", total, t_text, t_infer, result.get("classification")) + return PredictionResponse(**result) + + +@router.post("/predict/multimodal", response_model=PredictionResponse) +async def predict_multimodal(payload: MultimodalPayload) -> PredictionResponse: + if _inferencer is None: + raise HTTPException(status_code=503, detail="Inference model is not initialized.") + # Check route-level cache first + cache_key = _pred_cache_key(payload.model_dump()) + cached = _pred_cache_get(cache_key) + if cached is not None: + logger.debug("predict_multimodal cache HIT key=%s...", cache_key[:12]) + return PredictionResponse(**cached) + start = time.perf_counter() + audio = _apply_preprocessor(payload.audio_features, "audio") + video = _apply_preprocessor(payload.video_features, "video") + text = _apply_preprocessor(payload.text_features, "text") + sample = { + "audio": audio, + "video": video, + "text": text, + "audio_mask": payload.audio_mask if payload.audio_mask is not None else ([True] * len(audio) if audio else [False]), + "video_mask": payload.video_mask if payload.video_mask is not None else ([True] * len(video) if video else [False]), + "text_mask": payload.text_mask if payload.text_mask is not None else ([True] * len(text) if text else [False]), + } + result = _inferencer.predict_single(sample) + duration = time.perf_counter() - start + result.setdefault("metadata", {}) + result["processing_details"] = {"total_seconds": duration, "stages": {"inference": duration}} + _pred_cache_set(cache_key, result) + logger.info("predict_multimodal finished: duration=%.3fs classification=%.4f confidence=%.4f modality_scores=%s", duration, result.get("classification"), result.get("confidence"), result.get("modality_scores")) + return PredictionResponse(**result) + + +@router.post("/predict/question", response_model=PredictionResponse) +async def predict_question(payload: MultimodalPayload) -> PredictionResponse: + if _inferencer is None: + raise HTTPException(status_code=503, detail="Inference model is not initialized.") + start = time.perf_counter() + audio = _apply_preprocessor(payload.audio_features, "audio") + video = _apply_preprocessor(payload.video_features, "video") + text = _apply_preprocessor(payload.text_features, "text") + sample = { + "audio": audio, + "video": video, + "text": text, + "audio_mask": payload.audio_mask if payload.audio_mask is not None else ([True] * len(audio) if audio else [False]), + "video_mask": payload.video_mask if payload.video_mask is not None else ([True] * len(video) if video else [False]), + "text_mask": payload.text_mask if payload.text_mask is not None else ([True] * len(text) if text else [False]), + } + result = _inferencer.predict_single(sample) + duration = time.perf_counter() - start + result.setdefault("metadata", {}) + result["processing_details"] = {"total_seconds": duration, "stages": {"inference": duration}} + logger.info("predict_question finished: duration=%.3fs classification=%.4f", duration, result.get("classification")) + return PredictionResponse(**result) + + +@router.post("/predict/audio/raw", response_model=PredictionResponse) +async def predict_audio_raw( + file: UploadFile = File(...), + participant_id: str = Form("unknown"), +) -> PredictionResponse: + """Accept a raw audio file upload, extract MFCC features with librosa, and run inference. + + Used by the backend's predict_extended() call for per-question audio scoring. + Feature pipeline: MFCC(13) + delta + delta2 → [T, 39] → normalizer+PCA → [T, audio_dim]. + """ + if _inferencer is None: + raise HTTPException(status_code=503, detail="Inference model is not initialized.") + + audio_bytes = await file.read() + if not audio_bytes: + raise HTTPException(status_code=400, detail="Empty audio file received.") + + try: + import librosa + waveform, _ = librosa.load(io.BytesIO(audio_bytes), sr=16000, mono=True) + except Exception as exc: + raise HTTPException(status_code=400, detail=f"Failed to decode audio: {exc}") + + if len(waveform) < 320: + raise HTTPException(status_code=400, detail="Audio too short — need at least 20ms of audio.") + + waveform = librosa.util.normalize(waveform) + mfcc = librosa.feature.mfcc(y=waveform, sr=16000, n_mfcc=13) + delta = librosa.feature.delta(mfcc) + delta2 = librosa.feature.delta(mfcc, order=2) + features = np.vstack([mfcc, delta, delta2]).T.astype(np.float32) # [T, 39] + features = np.nan_to_num(features, nan=1e-6, posinf=1e-6, neginf=-1e-6) + + audio_features = _apply_preprocessor(features.tolist(), "audio") + + # Ensure output dim matches model's audio_input_dim regardless of preprocessor outcome + target_dim = _inferencer.model.audio_input_dim + arr = np.array(audio_features, dtype=np.float32) + if arr.shape[-1] != target_dim: + logger.warning( + "predict_audio_raw: dim mismatch after preprocessing — got %d, expected %d; adjusting", + arr.shape[-1], target_dim, + ) + if arr.shape[-1] > target_dim: + arr = arr[:, :target_dim] + else: + pad = np.zeros((arr.shape[0], target_dim - arr.shape[-1]), dtype=np.float32) + arr = np.concatenate([arr, pad], axis=-1) + audio_features = arr.tolist() + + start = time.perf_counter() + sample = { + "audio": audio_features, + "audio_mask": [True] * len(audio_features), + } + result = _inferencer.predict_single(sample) + duration = time.perf_counter() - start + result.setdefault("metadata", {}) + result["processing_details"] = { + "total_seconds": duration, + "stages": {"inference": duration}, + "participant_id": participant_id, + "audio_frames": len(audio_features), + } + logger.info( + "predict_audio_raw finished: frames=%d duration=%.3fs classification=%.4f confidence=%.4f", + len(audio_features), duration, result.get("classification"), result.get("confidence"), + ) + return PredictionResponse(**result) diff --git a/src/api/schemas.py b/src/api/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..1c78637da87d0d11eb696726af0100b5a941a419 --- /dev/null +++ b/src/api/schemas.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from typing import List, Optional +from pydantic import BaseModel, Field, field_validator + + +class HealthResponse(BaseModel): + status: str + model_loaded: bool + description: str + + +class AudioPayload(BaseModel): + audio_path: Optional[str] = None + audio_features: Optional[List[List[float]]] = None + metadata: Optional[dict] = None + + +class VideoPayload(BaseModel): + video_path: Optional[str] = None + openface_csv_path: Optional[str] = None + video_frames: Optional[List[List[int]]] = None + video_features: Optional[List[List[float]]] = None + metadata: Optional[dict] = None + + +class TextPayload(BaseModel): + transcript: Optional[str] = None + chunked_transcript: Optional[List[str]] = None + metadata: Optional[dict] = None + + +class MultimodalPayload(BaseModel): + audio_features: Optional[List[List[float]]] = None + video_features: Optional[List[List[float]]] = None + text_features: Optional[List[List[float]]] = None + audio_mask: Optional[List[bool]] = None + video_mask: Optional[List[bool]] = None + text_mask: Optional[List[bool]] = None + metadata: Optional[dict] = None + + +class PredictionResponse(BaseModel): + phq_total: float + phq_questions: List[float] + classification: float + confidence: float + modality_scores: dict + entropy: float + metadata: Optional[dict] = None + processing_details: Optional[dict] = None + + @field_validator("phq_questions") + @classmethod + def validate_phq_questions(cls, value: List[float]) -> List[float]: + if len(value) != 8: + raise ValueError("phq_questions must contain 8 values") + return value diff --git a/src/dataset/__init__.py b/src/dataset/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/dataset/__pycache__/__init__.cpython-312.pyc b/src/dataset/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a2989939d31feb5dea3d31f3a6830abb4610eebd Binary files /dev/null and b/src/dataset/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/dataset/__pycache__/builder.cpython-312.pyc b/src/dataset/__pycache__/builder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d20bc71bc38ccc8b97d0d3da82582b04b3678523 Binary files /dev/null and b/src/dataset/__pycache__/builder.cpython-312.pyc differ diff --git a/src/dataset/__pycache__/collate.cpython-312.pyc b/src/dataset/__pycache__/collate.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..69803d2faff6f982e2d33518546d181ae39b8b5a Binary files /dev/null and b/src/dataset/__pycache__/collate.cpython-312.pyc differ diff --git a/src/dataset/__pycache__/multimodal_dataset.cpython-312.pyc b/src/dataset/__pycache__/multimodal_dataset.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9564c8994fb4c1a720ffc2b6bea8ecd55f9463c2 Binary files /dev/null and b/src/dataset/__pycache__/multimodal_dataset.cpython-312.pyc differ diff --git a/src/dataset/builder.py b/src/dataset/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..1beab6f0ddccff66fdb1a24ed53cb46a0dc959e9 --- /dev/null +++ b/src/dataset/builder.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Dict, List, Optional + +import pandas as pd +import numpy as np + +from src.features.feature_store import load_features +from src.dataset.multimodal_dataset import MultimodalSample + + +class DatasetBuilder: + @staticmethod + def build_from_csv( + split_csv_path: Path, + feature_dir: Path, + id_column: str = "participant_id", + label_column: str = "phq_total", + modality_keys: Optional[List[str]] = None, + augment: bool = False, + **kwargs, + ) -> MultimodalSample: + df = pd.read_csv(split_csv_path) + df.columns = df.columns.str.lower().str.strip() + id_column = id_column.lower() + label_column = label_column.lower() + samples = [] + modality_keys = modality_keys or ["audio", "video", "text"] + for _, row in df.iterrows(): + participant_id = str(row[id_column]) + feature_path = feature_dir / f"{participant_id}.npz" + if not feature_path.exists(): + continue + features = load_features(feature_path) + phq_questions = np.asarray( + [row.get(f"phq_q{i}", 0.0) for i in range(1, 9)], + dtype=np.float32, + ) + classification = row.get("binary") + if classification is None: + classification = row.get("phq_binary", float(row[label_column] >= 10)) + + video_feature = None + if "video" in modality_keys: + if "video" in features: + video_feature = features["video"] + elif "video_openface" in features: + video_feature = features["video_openface"] + elif "video_cnn" in features: + video_feature = features["video_cnn"] + + sample = { + "participant_id": participant_id, + "audio": features.get("audio") if "audio" in modality_keys else None, + "video": video_feature, + "text": features.get("text") if "text" in modality_keys else None, + "phq_total": float(row[label_column]), + "phq_questions": phq_questions, + "classification": float(classification), + } + samples.append(sample) + return MultimodalSample(samples=samples, augment=augment, **kwargs) diff --git a/src/dataset/collate.py b/src/dataset/collate.py new file mode 100644 index 0000000000000000000000000000000000000000..b53bf0250f3f9c8402f5a3fb5da58bad3378e410 --- /dev/null +++ b/src/dataset/collate.py @@ -0,0 +1,68 @@ +from typing import Any, Dict, List +import torch + + +def _detect_dim(batch: List[Dict[str, Any]], key: str) -> int: + for item in batch: + value = item.get(key) + if value is not None and value.ndim == 2: + return value.shape[1] + return 0 + + +def multimodal_collate_fn(batch: List[Dict[str, Any]]) -> Dict[str, Any]: + batch_size = len(batch) + audio_lengths = [item["audio"].shape[0] if item["audio"] is not None else 0 for item in batch] + video_lengths = [item["video"].shape[0] if item["video"] is not None else 0 for item in batch] + text_lengths = [item["text"].shape[0] if item["text"] is not None else 0 for item in batch] + + max_audio = max(audio_lengths) if audio_lengths else 0 + max_video = max(video_lengths) if video_lengths else 0 + max_text = max(text_lengths) if text_lengths else 0 + + audio_dim = _detect_dim(batch, "audio") + video_dim = _detect_dim(batch, "video") + text_dim = _detect_dim(batch, "text") + + audio_tensor = torch.zeros(batch_size, max_audio, audio_dim, dtype=torch.float32) + video_tensor = torch.zeros(batch_size, max_video, video_dim, dtype=torch.float32) + text_tensor = torch.zeros(batch_size, max_text, text_dim, dtype=torch.float32) + audio_mask = torch.zeros(batch_size, max_audio, dtype=torch.bool) + video_mask = torch.zeros(batch_size, max_video, dtype=torch.bool) + text_mask = torch.zeros(batch_size, max_text, dtype=torch.bool) + + phq_totals = torch.zeros(batch_size, dtype=torch.float32) + phq_questions = torch.zeros(batch_size, 8, dtype=torch.float32) + classifications = torch.zeros(batch_size, dtype=torch.float32) + participant_ids = [] + + for i, item in enumerate(batch): + if item["audio"] is not None: + length = item["audio"].shape[0] + audio_tensor[i, :length] = item["audio"] + audio_mask[i, :length] = True + if item["video"] is not None: + length = item["video"].shape[0] + video_tensor[i, :length] = item["video"] + video_mask[i, :length] = True + if item["text"] is not None: + length = item["text"].shape[0] + text_tensor[i, :length] = item["text"] + text_mask[i, :length] = True + phq_totals[i] = item["phq_total"] + phq_questions[i] = item["phq_questions"] + classifications[i] = item["classification"] + participant_ids.append(item["participant_id"]) + + return { + "audio": audio_tensor, + "video": video_tensor, + "text": text_tensor, + "audio_mask": audio_mask, + "video_mask": video_mask, + "text_mask": text_mask, + "phq_total": phq_totals, + "phq_questions": phq_questions, + "classification": classifications, + "participant_ids": participant_ids, + } diff --git a/src/dataset/multimodal_dataset.py b/src/dataset/multimodal_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..302cc98dfbbcc8d9d8ab1ae9b3df6e17486671d1 --- /dev/null +++ b/src/dataset/multimodal_dataset.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import numpy as np +import torch +from pathlib import Path +from typing import Dict, List, Optional, Sequence, Any +from torch.utils.data import Dataset + +from src.features.feature_store import load_features +from src.features.sanitization import sanitize_array + + +class MultimodalSample(Dataset): + def __init__( + self, + samples: List[Dict[str, Any]], + augment: bool = False, + temporal_dropout_rate: float = 0.1, + feature_noise_std: float = 0.01, + ) -> None: + self.samples = samples + self.augment = augment + self.temporal_dropout_rate = temporal_dropout_rate + self.feature_noise_std = feature_noise_std + + def __len__(self) -> int: + return len(self.samples) + + def __getitem__(self, idx: int) -> Dict[str, Any]: + sample = self.samples[idx] + audio = self._prepare_feature(sample.get("audio")) + video = self._prepare_feature(sample.get("video")) + text = self._prepare_feature(sample.get("text")) + + if self.augment: + audio, video, text = self._augment(audio, video, text) + + return { + "participant_id": sample["participant_id"], + "audio": torch.from_numpy(audio) if audio is not None else None, + "video": torch.from_numpy(video) if video is not None else None, + "text": torch.from_numpy(text) if text is not None else None, + "audio_mask": torch.ones(audio.shape[0], dtype=torch.bool) if audio is not None else torch.zeros(0, dtype=torch.bool), + "video_mask": torch.ones(video.shape[0], dtype=torch.bool) if video is not None else torch.zeros(0, dtype=torch.bool), + "text_mask": torch.ones(text.shape[0], dtype=torch.bool) if text is not None else torch.zeros(0, dtype=torch.bool), + "phq_total": torch.tensor(sample["phq_total"], dtype=torch.float32), + "phq_questions": torch.tensor(sample["phq_questions"], dtype=torch.float32), + "classification": torch.tensor(sample["classification"], dtype=torch.float32), + } + + @staticmethod + def _prepare_feature(feature: Optional[np.ndarray]) -> Optional[np.ndarray]: + if feature is None: + return None + feature = sanitize_array(feature) + if feature.ndim == 1: + raise ValueError("Feature arrays must be 2D with shape [time, dim], but received 1D input.") + if feature.ndim != 2: + raise ValueError("Feature arrays must be 2D with shape [time, dim].") + return feature + + @staticmethod + def _temporal_dropout(feat: np.ndarray, rate: float) -> np.ndarray: + """Drop random time steps from a feature sequence.""" + if rate <= 0.0 or feat is None: + return feat + keep = np.random.rand(feat.shape[0]) > rate + if keep.sum() < 2: + return feat + return feat[keep] + + @staticmethod + def _freq_mask(feat: np.ndarray, max_mask_frac: float = 0.20) -> np.ndarray: + """SpecAugment-style: zero out a contiguous block of feature dimensions.""" + if feat is None or feat.shape[1] < 4: + return feat + n_dims = feat.shape[1] + mask_width = max(1, int(np.random.uniform(0, max_mask_frac) * n_dims)) + start = np.random.randint(0, n_dims - mask_width + 1) + feat = feat.copy() + feat[:, start: start + mask_width] = 0.0 + return feat + + @staticmethod + def _time_warp(feat: np.ndarray, warp_frac: float = 0.10) -> np.ndarray: + """Randomly sub-sample or repeat time steps to simulate speed variation.""" + if feat is None or feat.shape[0] < 4: + return feat + T = feat.shape[0] + new_T = max(2, int(T * np.random.uniform(1.0 - warp_frac, 1.0 + warp_frac))) + indices = np.round(np.linspace(0, T - 1, new_T)).astype(int).clip(0, T - 1) + return feat[indices] + + def _augment( + self, + audio: Optional[np.ndarray], + video: Optional[np.ndarray], + text: Optional[np.ndarray], + ) -> tuple[Optional[np.ndarray], Optional[np.ndarray], Optional[np.ndarray]]: + # Temporal dropout (random frame removal) + audio = self._temporal_dropout(audio, self.temporal_dropout_rate) + video = self._temporal_dropout(video, self.temporal_dropout_rate) + text = self._temporal_dropout(text, self.temporal_dropout_rate) + + # SpecAugment-style feature dimension masking + if np.random.rand() < 0.5: + audio = self._freq_mask(audio, max_mask_frac=0.20) + if np.random.rand() < 0.5: + video = self._freq_mask(video, max_mask_frac=0.20) + if np.random.rand() < 0.4: + text = self._freq_mask(text, max_mask_frac=0.15) + + # Time-warp jitter (speed perturbation) + if np.random.rand() < 0.5: + audio = self._time_warp(audio, warp_frac=0.10) + if np.random.rand() < 0.5: + video = self._time_warp(video, warp_frac=0.10) + + # Additive Gaussian noise + if self.feature_noise_std > 0.0: + if audio is not None: + audio = audio + np.random.normal(0, self.feature_noise_std, audio.shape).astype(np.float32) + if video is not None: + video = video + np.random.normal(0, self.feature_noise_std, video.shape).astype(np.float32) + if text is not None: + text = text + np.random.normal(0, self.feature_noise_std * 0.5, text.shape).astype(np.float32) + return audio, video, text + + def feature_dimensions(self) -> Dict[str, int]: + dims: Dict[str, int] = {"audio": 0, "video": 0, "text": 0} + for sample in self.samples: + for key in dims: + feature = sample.get(key) + if feature is not None and getattr(feature, "ndim", 0) == 2: + dims[key] = feature.shape[1] + if dims[key] > 0: + break + return dims + + @classmethod + def from_feature_store( + cls, + split_csv_path: Path, + feature_dir: Path, + label_column: str = "phq_total", + id_column: str = "participant_id", + modality_keys: Optional[Sequence[str]] = None, + **kwargs, + ) -> "MultimodalSample": + import pandas as pd + + df = pd.read_csv(split_csv_path) + samples = [] + modality_keys = modality_keys or ["audio", "video", "text"] + for _, row in df.iterrows(): + participant_id = str(row[id_column]) + feature_path = feature_dir / f"{participant_id}.npz" + if not feature_path.exists(): + continue + features = load_features(feature_path) + sample = { + "participant_id": participant_id, + "audio": features.get("audio") if "audio" in modality_keys else None, + "video": features.get("video") if "video" in modality_keys else None, + "text": features.get("text") if "text" in modality_keys else None, + "phq_total": float(row[label_column]), + "phq_questions": np.asarray( + [row.get(f"phq_q{i}", 0.0) for i in range(1, 9)], + dtype=np.float32, + ), + "classification": float(row.get("binary", float(row[label_column] >= 10))), + } + samples.append(sample) + return cls(samples=samples, **kwargs) diff --git a/src/evaluation/__init__.py b/src/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/evaluation/__pycache__/__init__.cpython-312.pyc b/src/evaluation/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a6d9ae5e5c996e0fe0d86af686ff0131a16974b8 Binary files /dev/null and b/src/evaluation/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/evaluation/__pycache__/evaluator.cpython-312.pyc b/src/evaluation/__pycache__/evaluator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e1ac54d9b42a0a5de861f980c5747ca3fdb6e477 Binary files /dev/null and b/src/evaluation/__pycache__/evaluator.cpython-312.pyc differ diff --git a/src/evaluation/evaluator.py b/src/evaluation/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..3449f9e6108cd249f3b535ecd4d741af80f3f380 --- /dev/null +++ b/src/evaluation/evaluator.py @@ -0,0 +1,27 @@ +import numpy as np +from typing import Sequence + +from src.training.metrics import regression_metrics, classification_metrics + + +def evaluate_predictions( + phq_total_true: Sequence[float], + phq_total_pred: Sequence[float], + phq_question_true: Sequence[Sequence[float]], + phq_question_pred: Sequence[Sequence[float]], + classification_true: Sequence[float], + classification_pred_probs: Sequence[float], +) -> dict[str, float]: + total_results = regression_metrics(np.array(phq_total_true), np.array(phq_total_pred)) + question_results = regression_metrics( + np.array(phq_question_true).reshape(-1), + np.array(phq_question_pred).reshape(-1), + ) + classification_results = classification_metrics( + np.array(classification_true), np.array(classification_pred_probs) + ) + return { + **{f"total_{k}": v for k, v in total_results.items()}, + **{f"question_{k}": v for k, v in question_results.items()}, + **classification_results, + } diff --git a/src/features/__init__.py b/src/features/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/features/__pycache__/__init__.cpython-312.pyc b/src/features/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..49f8656704999eee5a772b9b05fdef843188ec71 Binary files /dev/null and b/src/features/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/features/__pycache__/audio_features.cpython-312.pyc b/src/features/__pycache__/audio_features.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ccba44abc0db0e0629f2c3295050cf50076d806 Binary files /dev/null and b/src/features/__pycache__/audio_features.cpython-312.pyc differ diff --git a/src/features/__pycache__/feature_store.cpython-312.pyc b/src/features/__pycache__/feature_store.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4eb9af9c4346e5e14e4bfee981e4ea7a96ac0106 Binary files /dev/null and b/src/features/__pycache__/feature_store.cpython-312.pyc differ diff --git a/src/features/__pycache__/pca_transform.cpython-312.pyc b/src/features/__pycache__/pca_transform.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8feff5c33779e5c5699e39fd7fc40f046968744b Binary files /dev/null and b/src/features/__pycache__/pca_transform.cpython-312.pyc differ diff --git a/src/features/__pycache__/preprocessing.cpython-312.pyc b/src/features/__pycache__/preprocessing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5e3cf54da77662d22e9fe55ba070fe9181c333e5 Binary files /dev/null and b/src/features/__pycache__/preprocessing.cpython-312.pyc differ diff --git a/src/features/__pycache__/sanitization.cpython-312.pyc b/src/features/__pycache__/sanitization.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08f03e4aea875a54b1c44774ba488121eceacee4 Binary files /dev/null and b/src/features/__pycache__/sanitization.cpython-312.pyc differ diff --git a/src/features/__pycache__/specs.cpython-312.pyc b/src/features/__pycache__/specs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2fe75f077bce4ed393b6cf907a03b5eb6f88c980 Binary files /dev/null and b/src/features/__pycache__/specs.cpython-312.pyc differ diff --git a/src/features/__pycache__/text_features.cpython-312.pyc b/src/features/__pycache__/text_features.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f3841eb2baedae70ec1a371bbb5a6c7f5054a47 Binary files /dev/null and b/src/features/__pycache__/text_features.cpython-312.pyc differ diff --git a/src/features/__pycache__/video_features.cpython-312.pyc b/src/features/__pycache__/video_features.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..308cd312d827f6415866465aed340768200582cc Binary files /dev/null and b/src/features/__pycache__/video_features.cpython-312.pyc differ diff --git a/src/features/audio_features.py b/src/features/audio_features.py new file mode 100644 index 0000000000000000000000000000000000000000..b5e46f42e6e108d2f8c157b63dbf30f27df5a264 --- /dev/null +++ b/src/features/audio_features.py @@ -0,0 +1,71 @@ +import math +from pathlib import Path +from typing import Dict, Optional + +import librosa +import numpy as np +import torch +from transformers import Wav2Vec2FeatureExtractor, Wav2Vec2Model + +from src.features.sanitization import sanitize_array + + +class AudioFeatureExtractor: + """Audio feature extraction for train and inference. + + Supports classical features (MFCC, delta, silence ratio) and + modern self-supervised embeddings (HuBERT/Wav2Vec2). + """ + + def __init__(self, config: dict): + self.sample_rate = config.get("sample_rate", 16000) + self.mfcc_n_mfcc = config.get("mfcc_n_mfcc", 13) + self.hubert_model_name = config.get("hubert_model_name", "facebook/hubert-base-ls960") + self.feature_backends = tuple(config.get("feature_backends", ["mfcc", "hubert"])) + + self._hubert_extractor: Optional[Wav2Vec2FeatureExtractor] = None + self._hubert_model: Optional[Wav2Vec2Model] = None + + def _ensure_hubert(self): + if self._hubert_model is None: + self._hubert_extractor = Wav2Vec2FeatureExtractor.from_pretrained(self.hubert_model_name) + self._hubert_model = Wav2Vec2Model.from_pretrained(self.hubert_model_name) + self._hubert_model.eval() + + def extract_classical(self, waveform: np.ndarray) -> Dict[str, np.ndarray]: + waveform = waveform.astype(np.float32) + waveform = librosa.util.normalize(waveform) + mfcc = librosa.feature.mfcc(y=waveform, sr=self.sample_rate, n_mfcc=self.mfcc_n_mfcc) + delta = librosa.feature.delta(mfcc) + delta2 = librosa.feature.delta(mfcc, order=2) + energy = librosa.feature.rms(y=waveform)[0] + silence_ratio = float(np.mean(energy < np.percentile(energy, 10))) + speech_rate = float(np.sum(energy > np.median(energy)) / max(1, len(energy))) + + return { + "mfcc": sanitize_array(mfcc.T), + "delta": sanitize_array(delta.T), + "delta2": sanitize_array(delta2.T), + "silence_ratio": np.array([silence_ratio], dtype=np.float32), + "speech_rate": np.array([speech_rate], dtype=np.float32), + } + + def extract_modern(self, waveform: np.ndarray) -> np.ndarray: + self._ensure_hubert() + waveform = waveform.astype(np.float32) + if waveform.ndim > 1: + waveform = waveform.mean(axis=-1) + inputs = self._hubert_extractor(waveform, sampling_rate=self.sample_rate, return_tensors="pt", padding=True) + with torch.no_grad(): + outputs = self._hubert_model(**inputs) + embedding = outputs.last_hidden_state.mean(dim=1) + return sanitize_array(embedding.squeeze(0).cpu().numpy()) + + def extract_features(self, waveform: np.ndarray) -> Dict[str, np.ndarray]: + features: Dict[str, np.ndarray] = {} + if "mfcc" in self.feature_backends: + classical = self.extract_classical(waveform) + features.update(classical) + if "hubert" in self.feature_backends: + features["hubert"] = self.extract_modern(waveform) + return features diff --git a/src/features/feature_store.py b/src/features/feature_store.py new file mode 100644 index 0000000000000000000000000000000000000000..e94ba5f5a80aa98b33107140bcbd307284e5b147 --- /dev/null +++ b/src/features/feature_store.py @@ -0,0 +1,13 @@ +import numpy as np +from pathlib import Path +from typing import Any, Dict + + +def save_features(path: Path, features: Dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(path, **features) + + +def load_features(path: Path) -> Dict[str, np.ndarray]: + loaded = np.load(path, allow_pickle=True) + return {key: loaded[key] for key in loaded.files} diff --git a/src/features/pca_transform.py b/src/features/pca_transform.py new file mode 100644 index 0000000000000000000000000000000000000000..c671c274566ca7b006d86d9bd08efbaf57233b36 --- /dev/null +++ b/src/features/pca_transform.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import pickle +from pathlib import Path + +import numpy as np +from sklearn.decomposition import PCA + + +class PCAReducer: + def __init__(self, n_components: int = 64, whiten: bool = False): + self.n_components = n_components + self.whiten = whiten + self._pca: PCA | None = None + + def fit(self, data: np.ndarray) -> "PCAReducer": + self._pca = PCA(n_components=self.n_components, whiten=self.whiten) + self._pca.fit(data) + return self + + def transform(self, data: np.ndarray) -> np.ndarray: + if self._pca is None: + raise ValueError("PCA reducer must be fit before calling transform()") + return self._pca.transform(data) + + def fit_transform(self, data: np.ndarray) -> np.ndarray: + self.fit(data) + return self.transform(data) + + def save(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "wb") as handle: + pickle.dump({"n_components": self.n_components, "whiten": self.whiten, "pca": self._pca}, handle) + + @classmethod + def load(cls, path: Path) -> "PCAReducer": + with open(path, "rb") as handle: + payload = pickle.load(handle) + reducer = cls(n_components=payload["n_components"], whiten=payload["whiten"]) + reducer._pca = payload["pca"] + return reducer diff --git a/src/features/preprocessing.py b/src/features/preprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..05b0e164affc33ba4ec6af76a2f0eb9affa11c5b --- /dev/null +++ b/src/features/preprocessing.py @@ -0,0 +1,295 @@ +"""Robust feature normalization and PCA transformation with serialization. + +This module provides standardized normalization and PCA transforms that ensure +identical preprocessing between training and inference. +""" + +from __future__ import annotations + +import pickle +from pathlib import Path +from typing import Optional, Tuple +import numpy as np +from sklearn.decomposition import PCA +from sklearn.preprocessing import StandardScaler, RobustScaler, MinMaxScaler + + +class NormalizerRegistry: + """Registry of supported normalization methods.""" + + _registry = { + "standard": StandardScaler, + "robust": RobustScaler, + "minmax": MinMaxScaler, + } + + @classmethod + def get(cls, name: str) -> type: + """Get normalizer class by name.""" + if name not in cls._registry: + raise ValueError(f"Unknown normalizer: {name}. Available: {list(cls._registry.keys())}") + return cls._registry[name] + + @classmethod + def register(cls, name: str, normalizer_class: type) -> None: + """Register a custom normalizer.""" + cls._registry[name] = normalizer_class + + +class FeatureNormalizer: + """Fit and apply feature normalization with serialization.""" + + def __init__(self, normalizer_type: str = "standard", epsilon: float = 1e-8): + """Initialize normalizer. + + Args: + normalizer_type: Type of normalizer ("standard", "robust", "minmax") + epsilon: Small value to prevent division by zero + """ + self.normalizer_type = normalizer_type + self.epsilon = epsilon + normalizer_class = NormalizerRegistry.get(normalizer_type) + self._normalizer = normalizer_class() + self._fitted = False + + def fit(self, features: np.ndarray) -> FeatureNormalizer: + """Fit normalizer to features.""" + features = np.asarray(features, dtype=np.float32) + if features.ndim == 3: # Batch of temporal features [batch, time, dim] + features = features.reshape(-1, features.shape[-1]) + elif features.ndim == 2: # Single or multiple [time, dim] + pass + else: + raise ValueError(f"Expected 2D or 3D features, got {features.ndim}D") + + self._normalizer.fit(features) + self._fitted = True + return self + + def transform(self, features: np.ndarray) -> np.ndarray: + """Apply normalization.""" + if not self._fitted: + raise ValueError("Normalizer must be fit before calling transform()") + + features = np.asarray(features, dtype=np.float32) + if features.ndim == 2: + return self._normalizer.transform(features).astype(np.float32) + elif features.ndim == 3: + batch_size, time_steps, dim = features.shape + reshaped = features.reshape(-1, dim) + transformed = self._normalizer.transform(reshaped) + return transformed.reshape(batch_size, time_steps, dim).astype(np.float32) + else: + raise ValueError(f"Expected 2D or 3D features, got {features.ndim}D") + + def fit_transform(self, features: np.ndarray) -> np.ndarray: + """Fit and transform features.""" + self.fit(features) + return self.transform(features) + + def save(self, path: Path) -> None: + """Save normalizer to disk.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "wb") as f: + pickle.dump({ + "normalizer_type": self.normalizer_type, + "epsilon": self.epsilon, + "normalizer": self._normalizer, + "fitted": self._fitted, + }, f) + + @classmethod + def load(cls, path: Path) -> FeatureNormalizer: + """Load normalizer from disk.""" + with open(path, "rb") as f: + payload = pickle.load(f) + + normalizer = cls( + normalizer_type=payload["normalizer_type"], + epsilon=payload["epsilon"], + ) + normalizer._normalizer = payload["normalizer"] + normalizer._fitted = payload["fitted"] + return normalizer + + +class PCATransform: + """Fit and apply PCA transformation with serialization.""" + + def __init__(self, n_components: int | float = 0.95, whiten: bool = False): + """Initialize PCA transform. + + Args: + n_components: Number of principal components to keep, or float for variance ratio + whiten: Whether to apply whitening + """ + self.n_components = n_components + self.whiten = whiten + self._pca: Optional[PCA] = None + self._fitted = False + + def fit(self, features: np.ndarray) -> PCATransform: + """Fit PCA to features.""" + features = np.asarray(features, dtype=np.float32) + if features.ndim == 3: # Batch of temporal features [batch, time, dim] + features = features.reshape(-1, features.shape[-1]) + elif features.ndim == 2: + pass + else: + raise ValueError(f"Expected 2D or 3D features, got {features.ndim}D") + + self._pca = PCA(n_components=self.n_components, whiten=self.whiten) + self._pca.fit(features) + self._fitted = True + return self + + def transform(self, features: np.ndarray) -> np.ndarray: + """Apply PCA transformation.""" + if not self._fitted or self._pca is None: + raise ValueError("PCA must be fit before calling transform()") + + features = np.asarray(features, dtype=np.float32) + original_shape = None + + if features.ndim == 3: # Batch of temporal features + batch_size, time_steps, dim = features.shape + original_shape = (batch_size, time_steps) + features = features.reshape(-1, dim) + elif features.ndim == 2: + pass + else: + raise ValueError(f"Expected 2D or 3D features, got {features.ndim}D") + + transformed = self._pca.transform(features).astype(np.float32) + + if original_shape is not None: + transformed = transformed.reshape(original_shape[0], original_shape[1], -1) + + return transformed + + def fit_transform(self, features: np.ndarray) -> np.ndarray: + """Fit and transform features.""" + self.fit(features) + return self.transform(features) + + @property + def explained_variance_ratio(self) -> Optional[np.ndarray]: + """Get explained variance ratio.""" + return self._pca.explained_variance_ratio_ if self._fitted and self._pca else None + + @property + def total_variance_explained(self) -> Optional[float]: + """Get total variance explained.""" + if self.explained_variance_ratio is not None: + return float(self.explained_variance_ratio.sum()) + return None + + def save(self, path: Path) -> None: + """Save PCA transform to disk.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "wb") as f: + pickle.dump({ + "n_components": self.n_components, + "whiten": self.whiten, + "pca": self._pca, + "fitted": self._fitted, + }, f) + + @classmethod + def load(cls, path: Path) -> PCATransform: + """Load PCA transform from disk.""" + with open(path, "rb") as f: + payload = pickle.load(f) + + transform = cls( + n_components=payload["n_components"], + whiten=payload["whiten"], + ) + transform._pca = payload["pca"] + transform._fitted = payload["fitted"] + return transform + + +class FeaturePreprocessor: + """Unified feature preprocessing: normalization + optional PCA.""" + + def __init__( + self, + normalizer_type: str = "standard", + apply_pca: bool = False, + pca_components: int | float = 0.95, + pca_whiten: bool = False, + ): + """Initialize preprocessor. + + Args: + normalizer_type: Type of normalization ("standard", "robust", "minmax") + apply_pca: Whether to apply PCA + pca_components: Number of PCA components if apply_pca=True + pca_whiten: Whether to apply whitening in PCA + """ + self.normalizer_type = normalizer_type + self.apply_pca = apply_pca + self.pca_components = pca_components + self.pca_whiten = pca_whiten + + self.normalizer = FeatureNormalizer(normalizer_type=normalizer_type) + self.pca: Optional[PCATransform] = None + if apply_pca: + self.pca = PCATransform(n_components=pca_components, whiten=pca_whiten) + + def fit(self, features: np.ndarray) -> FeaturePreprocessor: + """Fit normalizer and PCA.""" + self.normalizer.fit(features) + if self.apply_pca and self.pca: + normalized = self.normalizer.transform(features) + self.pca.fit(normalized) + return self + + def transform(self, features: np.ndarray) -> np.ndarray: + """Apply normalization and PCA.""" + normalized = self.normalizer.transform(features) + if self.apply_pca and self.pca: + return self.pca.transform(normalized) + return normalized + + def fit_transform(self, features: np.ndarray) -> np.ndarray: + """Fit and transform features.""" + self.fit(features) + return self.transform(features) + + @property + def output_dim(self) -> int: + """Get output feature dimension.""" + if self.apply_pca and self.pca and self.pca._fitted: + return self.pca._pca.n_components_ + return None # Unknown without fitting + + def save(self, path: Path) -> None: + """Save preprocessor to disk.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "wb") as f: + pickle.dump({ + "normalizer_type": self.normalizer_type, + "apply_pca": self.apply_pca, + "pca_components": self.pca_components, + "pca_whiten": self.pca_whiten, + "normalizer": self.normalizer, + "pca": self.pca, + }, f) + + @classmethod + def load(cls, path: Path) -> FeaturePreprocessor: + """Load preprocessor from disk.""" + with open(path, "rb") as f: + payload = pickle.load(f) + + preprocessor = cls( + normalizer_type=payload["normalizer_type"], + apply_pca=payload["apply_pca"], + pca_components=payload["pca_components"], + pca_whiten=payload["pca_whiten"], + ) + preprocessor.normalizer = payload["normalizer"] + preprocessor.pca = payload["pca"] + return preprocessor diff --git a/src/features/sanitization.py b/src/features/sanitization.py new file mode 100644 index 0000000000000000000000000000000000000000..debd66eb26610ca990268802a9f7601df89309fe --- /dev/null +++ b/src/features/sanitization.py @@ -0,0 +1,10 @@ +import numpy as np + + +def sanitize_array(array: np.ndarray, fill_value: float = 0.0) -> np.ndarray: + if not isinstance(array, np.ndarray): + array = np.asarray(array, dtype=np.float32) + else: + array = array.astype(np.float32, copy=False) + array = np.nan_to_num(array, nan=fill_value, posinf=fill_value, neginf=fill_value) + return array diff --git a/src/features/specs.py b/src/features/specs.py new file mode 100644 index 0000000000000000000000000000000000000000..68ea1e9638f312226d37ebe00fecf4407e5a5bf9 --- /dev/null +++ b/src/features/specs.py @@ -0,0 +1,254 @@ +"""Feature specification and validation system for training/inference consistency. + +This module defines strict feature specifications for each modality to ensure +training and inference produce identical feature representations. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, Optional +import numpy as np + + +@dataclass +class ModalitySpec: + """Specification for a single modality's features.""" + + name: str + expected_dim: int + temporal: bool = True + preprocessor_class: Optional[str] = None + extractor_class: Optional[str] = None + pca_enabled: bool = False + pca_target_dim: Optional[int] = None + normalizer_type: str = "standard" # "standard", "robust", "minmax", "none" + + def validate_features(self, features: np.ndarray) -> bool: + """Validate that features match specification.""" + if not isinstance(features, np.ndarray): + return False + if features.dtype != np.float32: + return False + if self.temporal: + if features.ndim != 2: + return False + if features.shape[1] != self.expected_dim: + return False + else: + if features.ndim != 1: + return False + if features.shape[0] != self.expected_dim: + return False + return not (np.isnan(features).any() or np.isinf(features).any()) + + def to_dict(self) -> Dict: + """Serialize spec to dict.""" + return { + "name": self.name, + "expected_dim": self.expected_dim, + "temporal": self.temporal, + "preprocessor_class": self.preprocessor_class, + "extractor_class": self.extractor_class, + "pca_enabled": self.pca_enabled, + "pca_target_dim": self.pca_target_dim, + "normalizer_type": self.normalizer_type, + } + + @classmethod + def from_dict(cls, data: Dict) -> ModalitySpec: + """Deserialize spec from dict.""" + return cls(**data) + + +class AudioSpec: + """Audio modality specifications.""" + + # MFCC-based audio: 13 MFCC + 13 delta + 13 delta2 = 39 + MFCC_DIM = 39 + + # HuBERT-base embeddings + HUBERT_DIM = 768 + + # Combined features: MFCC (39) + HuBERT (768) = 807 + COMBINED_DIM = MFCC_DIM + HUBERT_DIM + + # Standard specification: use MFCC features for training + STANDARD = ModalitySpec( + name="audio", + expected_dim=MFCC_DIM, # 39: 13 MFCC + 13 delta + 13 delta2 + temporal=True, + preprocessor_class="AudioPreprocessor", + extractor_class="AudioFeatureExtractor", + pca_enabled=False, + normalizer_type="standard", + ) + + # Optional: HuBERT embeddings + HUBERT = ModalitySpec( + name="audio_hubert", + expected_dim=HUBERT_DIM, # 768 + temporal=True, + preprocessor_class="AudioPreprocessor", + extractor_class="AudioFeatureExtractor", + pca_enabled=True, + pca_target_dim=128, + normalizer_type="standard", + ) + + # Optional: Combined MFCC + HuBERT + COMBINED = ModalitySpec( + name="audio_combined", + expected_dim=COMBINED_DIM, # 807 + temporal=True, + preprocessor_class="AudioPreprocessor", + extractor_class="AudioFeatureExtractor", + pca_enabled=True, + pca_target_dim=128, + normalizer_type="standard", + ) + + +class VideoSpec: + """Video modality specifications.""" + + # OpenFace features: AU (17) + gaze (3) + head pose (3) + landmarks dynamics (many) + OPENFACE_DIM = 38 # Conservative estimate for behavioral features + + # ResNet50 embeddings: 2048D, typically PCA-reduced + RESNET_RAW_DIM = 2048 + RESNET_PCA_DIM = 128 # Standard reduction for ResNet + + # Specification: behavioral features + visual embeddings + BEHAVIORAL = ModalitySpec( + name="video_behavior", + expected_dim=OPENFACE_DIM, # 38: OpenFace AU + gaze + pose + temporal=True, + preprocessor_class="VideoPreprocessor", + extractor_class="VideoFeatureExtractor", + pca_enabled=False, + normalizer_type="standard", + ) + + VISUAL_EMBEDDING = ModalitySpec( + name="video_embedding", + expected_dim=RESNET_PCA_DIM, # 128: ResNet50 PCA-reduced + temporal=True, + preprocessor_class="VideoPreprocessor", + extractor_class="VideoFeatureExtractor", + pca_enabled=True, # PCA is applied during extraction + pca_target_dim=RESNET_PCA_DIM, + normalizer_type="standard", + ) + + # Combined: behavioral + visual + COMBINED = ModalitySpec( + name="video_combined", + expected_dim=OPENFACE_DIM + RESNET_PCA_DIM, # 166: behavioral + visual + temporal=True, + preprocessor_class="VideoPreprocessor", + extractor_class="VideoFeatureExtractor", + pca_enabled=False, + normalizer_type="standard", + ) + + # Standard: behavioral features only (recommended for small datasets) + STANDARD = BEHAVIORAL + + +class TextSpec: + """Text modality specifications.""" + + # SBERT all-mpnet-base-v2 embeddings + SBERT_DIM = 384 + + # Standard specification + STANDARD = ModalitySpec( + name="text", + expected_dim=SBERT_DIM, # 384: SBERT embeddings + temporal=True, # Multiple chunks per session + preprocessor_class="TextPreprocessor", + extractor_class="TextFeatureExtractor", + pca_enabled=False, + normalizer_type="standard", + ) + + +class FeatureSpecification: + """Complete multimodal feature specification for training/inference consistency.""" + + def __init__( + self, + audio_spec: ModalitySpec = AudioSpec.STANDARD, + video_spec: ModalitySpec = VideoSpec.STANDARD, + text_spec: ModalitySpec = TextSpec.STANDARD, + ): + self.audio_spec = audio_spec + self.video_spec = video_spec + self.text_spec = text_spec + self._specs = { + "audio": audio_spec, + "video": video_spec, + "text": text_spec, + } + + @property + def audio_dim(self) -> int: + return self.audio_spec.expected_dim + + @property + def video_dim(self) -> int: + return self.video_spec.expected_dim + + @property + def text_dim(self) -> int: + return self.text_spec.expected_dim + + def get_spec(self, modality: str) -> ModalitySpec: + """Get specification for a modality.""" + if modality not in self._specs: + raise ValueError(f"Unknown modality: {modality}") + return self._specs[modality] + + def validate(self, features: Dict[str, np.ndarray]) -> bool: + """Validate all feature modalities match specification.""" + for modality, feature_array in features.items(): + if modality in self._specs: + if not self._specs[modality].validate_features(feature_array): + return False + return True + + def to_dict(self) -> Dict: + """Serialize specification.""" + return { + "audio": self.audio_spec.to_dict(), + "video": self.video_spec.to_dict(), + "text": self.text_spec.to_dict(), + } + + @classmethod + def default(cls) -> FeatureSpecification: + """Create default specification: MFCC audio + behavioral video + SBERT text.""" + return cls( + audio_spec=AudioSpec.STANDARD, + video_spec=VideoSpec.STANDARD, + text_spec=TextSpec.STANDARD, + ) + + @classmethod + def with_hubert(cls) -> FeatureSpecification: + """Create specification with HuBERT audio embeddings.""" + return cls( + audio_spec=AudioSpec.HUBERT, + video_spec=VideoSpec.STANDARD, + text_spec=TextSpec.STANDARD, + ) + + @classmethod + def multimodal_combined(cls) -> FeatureSpecification: + """Create specification with all combined features.""" + return cls( + audio_spec=AudioSpec.COMBINED, + video_spec=VideoSpec.COMBINED, + text_spec=TextSpec.STANDARD, + ) diff --git a/src/features/text_features.py b/src/features/text_features.py new file mode 100644 index 0000000000000000000000000000000000000000..afdbae4453513fc236cf0fc186ae7fd8857e7983 --- /dev/null +++ b/src/features/text_features.py @@ -0,0 +1,63 @@ +from pathlib import Path +from typing import Dict, Sequence + +import numpy as np +from sentence_transformers import SentenceTransformer +from pathlib import Path +from typing import Optional +from src.features.preprocessing import FeaturePreprocessor, PCATransform +import logging +logger = logging.getLogger(__name__) + +from src.features.sanitization import sanitize_array + + +class TextFeatureExtractor: + """SentenceTransformer-based text embedding for transcripts. + + Optionally applies a serialized `FeaturePreprocessor` (normalizer + PCA) + saved during training so the runtime embeddings match the model input + dimensionality (e.g. 768 -> 133). + """ + + def __init__(self, model_name: str = "sentence-transformers/all-mpnet-base-v2", preprocessor_path: Optional[Path] = None): + self.model = SentenceTransformer(model_name) + self.preprocessor: Optional[FeaturePreprocessor] = None + if preprocessor_path is not None: + p = Path(preprocessor_path) + if p.exists(): + try: + self.preprocessor = FeaturePreprocessor.load(p) + logger.info("Loaded FeaturePreprocessor from %s", str(p)) + except Exception: + try: + # Try loading a raw PCA reducer if present + pca = PCATransform.load(p) + class _PCAWrap: + def __init__(self, pca): + self._pca = pca + def transform(self, x): + return self._pca.transform(x) + self.preprocessor = _PCAWrap(pca) + logger.info("Loaded PCATransform from %s", str(p)) + except Exception: + logger.warning("Failed to load preprocessor from %s; continuing without it", str(p)) + self.preprocessor = None + + def encode(self, texts: Sequence[str]) -> np.ndarray: + embeddings = self.model.encode(list(texts), convert_to_numpy=True, show_progress_bar=False) + arr = np.asarray(embeddings, dtype=np.float32) + if self.preprocessor is not None: + try: + arr = self.preprocessor.transform(arr) + except Exception: + pass + return sanitize_array(arr) + + def encode_text(self, transcript: str, chunks: Sequence[str]) -> Dict[str, np.ndarray]: + session_embedding = self.encode([transcript])[0] + chunk_embeddings = self.encode(chunks) + return { + "session": session_embedding, + "chunks": chunk_embeddings, + } diff --git a/src/features/video_features.py b/src/features/video_features.py new file mode 100644 index 0000000000000000000000000000000000000000..4ba3e9d70275e36056c15e8d5ce287899094b488 --- /dev/null +++ b/src/features/video_features.py @@ -0,0 +1,78 @@ +import csv +from pathlib import Path +from typing import Dict, List, Optional + +import numpy as np +import torch +import torchvision.models as models +import torchvision.transforms as T + +from src.features.sanitization import sanitize_array + + +class OpenFaceFeatureLoader: + """Load OpenFace CSV features for consistent train and inference.""" + + def __init__(self, csv_path: Path): + self.csv_path = csv_path + + def load(self) -> Dict[str, np.ndarray]: + with open(self.csv_path, "r", encoding="utf-8") as handle: + reader = csv.DictReader(handle) + rows = list(reader) + + if not rows: + raise ValueError(f"OpenFace CSV is empty: {self.csv_path}") + + values = np.array([[float(v or 0.0) for v in row.values()] for row in rows], dtype=np.float32) + columns = list(rows[0].keys()) + return { + "features": sanitize_array(values), + "columns": np.array(columns, dtype=object), + } + + +class VideoEmbeddingExtractor: + """Visual embedding backend for ResNet50 or placeholder VideoMAE.""" + + def __init__(self, backend: str = "resnet50", device: str = "cpu"): + self.backend = backend + self.device = device + if backend == "resnet50": + self.model = models.resnet50(pretrained=True).to(device).eval() + self.model = torch.nn.Sequential(*list(self.model.children())[:-1]) + else: + raise ValueError(f"Unsupported video embedding backend: {backend}") + self.transform = T.Compose([ + T.Resize((224, 224)), + T.ToTensor(), + T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ]) + + def embed_frames(self, frames: List[np.ndarray]) -> np.ndarray: + tensors = [self.transform(frame).to(self.device) for frame in frames] + batch = torch.stack(tensors, dim=0) + with torch.no_grad(): + embeddings = self.model(batch).squeeze(-1).squeeze(-1) + return sanitize_array(embeddings.cpu().numpy()) + + +class VideoFeatureExtractor: + """Combine behavioral OpenFace features with visual embedding features.""" + + def __init__(self, config: dict, device: str = "cpu"): + self.loader_backend = config.get("openface_csv_key", "openface") + self.embedding_backend = config.get("embedding_backend", "resnet50") + self.embedding_extractor = VideoEmbeddingExtractor(self.embedding_backend, device=device) + + def extract_openface(self, csv_path: Path) -> Dict[str, np.ndarray]: + loader = OpenFaceFeatureLoader(csv_path) + loaded = loader.load() + features = loaded["features"] + return { + "behavior": features, + "columns": loaded["columns"], + } + + def embed_frames(self, frames: List[np.ndarray]) -> np.ndarray: + return self.embedding_extractor.embed_frames(frames) diff --git a/src/inference/__init__.py b/src/inference/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/inference/__pycache__/__init__.cpython-312.pyc b/src/inference/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0e7dc43fa6288824f00b79e9251b9f714698e0cd Binary files /dev/null and b/src/inference/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/inference/__pycache__/inferencer.cpython-312.pyc b/src/inference/__pycache__/inferencer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88f53c47d89bd7be07f82c4c9bb2ed7b8897fe66 Binary files /dev/null and b/src/inference/__pycache__/inferencer.cpython-312.pyc differ diff --git a/src/inference/__pycache__/model_io.cpython-312.pyc b/src/inference/__pycache__/model_io.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..510528600bb86992a58b47434b734b2fcca7af87 Binary files /dev/null and b/src/inference/__pycache__/model_io.cpython-312.pyc differ diff --git a/src/inference/__pycache__/validation.cpython-312.pyc b/src/inference/__pycache__/validation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f19dac0ce26acd065ece980ee139d696348958aa Binary files /dev/null and b/src/inference/__pycache__/validation.cpython-312.pyc differ diff --git a/src/inference/inferencer.py b/src/inference/inferencer.py new file mode 100644 index 0000000000000000000000000000000000000000..e2d4807b757cc431e4bdde05b48ecb3da9cc3032 --- /dev/null +++ b/src/inference/inferencer.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Optional + +import numpy as np +import torch +from torch.utils.data import DataLoader + +from src.inference.model_io import load_checkpoint, sanitize_payload +from src.models.multimodal_model import MultimodalDepressionModel +from src.dataset.collate import multimodal_collate_fn + + +class ModelV2Inferencer: + def __init__(self, checkpoint_path: Path, model_config: Dict[str, Any], device: str = "auto"): + if device == "auto": + device = "cuda" if torch.cuda.is_available() else "cpu" + self.device = torch.device(device) + self.model = MultimodalDepressionModel(**model_config) + load_checkpoint(self.model, checkpoint_path, device=str(self.device)) + self.model.to(self.device).eval() + + def _prepare_mask(self, values: Any, mask: Any) -> torch.Tensor: + if mask is not None: + tensor_mask = torch.tensor(mask, dtype=torch.bool, device=self.device) + if tensor_mask.dim() == 1: + tensor_mask = tensor_mask.unsqueeze(0) + return tensor_mask + + if values is not None: + length = values.shape[0] + return torch.ones((1, length), dtype=torch.bool, device=self.device) + + return torch.zeros((1, 1), dtype=torch.bool, device=self.device) + + def predict_single(self, sample: Dict[str, Any]) -> Dict[str, Any]: + sample = sanitize_payload(sample) + with torch.no_grad(): + audio = sample.get("audio") + video = sample.get("video") + text = sample.get("text") + if audio is not None: + audio = torch.tensor(audio, dtype=torch.float32, device=self.device).unsqueeze(0) + if video is not None: + video = torch.tensor(video, dtype=torch.float32, device=self.device).unsqueeze(0) + if text is not None: + text = torch.tensor(text, dtype=torch.float32, device=self.device).unsqueeze(0) + + audio_mask = self._prepare_mask(audio, sample.get("audio_mask")) + video_mask = self._prepare_mask(video, sample.get("video_mask")) + text_mask = self._prepare_mask(text, sample.get("text_mask")) + + output = self.model(audio, video, text, audio_mask, video_mask, text_mask) + return { + "phq_total": float(output["phq_total"].cpu().numpy().mean()), + "phq_questions": output["phq_questions"].cpu().numpy().flatten().tolist(), + "classification": float(torch.sigmoid(output["classification"]).cpu().numpy().mean()), + "confidence": float(output["confidence"].cpu().numpy().mean()), + "modality_scores": {k: float(v.cpu().numpy().mean()) for k, v in output["modality_scores"].items()}, + "entropy": float(output["entropy"].cpu().numpy().mean()), + } + + def predict_batch(self, batch: list[Dict[str, Any]]) -> list[Dict[str, Any]]: + results = [] + for sample in batch: + results.append(self.predict_single(sample)) + return results diff --git a/src/inference/live_pipeline.py b/src/inference/live_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..9851a5a62d53cd03cac4914394fbbf57e893dd85 --- /dev/null +++ b/src/inference/live_pipeline.py @@ -0,0 +1,381 @@ +"""Live inference pipeline for real-time multimodal behavioral prediction. + +Handles end-to-end inference from raw inputs to model predictions with +full consistency with training pipeline. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Optional +import numpy as np +import torch +from torch.utils.data import DataLoader + +from src.inference.model_io import load_checkpoint, sanitize_payload +from src.inference.runtime_extractors import ( + LiveInferenceFeatureExtractor, + OpenFaceRuntimeExtractor, + AudioFeatureRuntimeExtractor, + TextFeatureRuntimeExtractor, +) +from src.inference.validation import ( + FeatureValidator, + TrainInferenceConsistencyChecker, +) +from src.models.multimodal_model import MultimodalDepressionModel +from src.features.specs import FeatureSpecification +from src.features.preprocessing import FeaturePreprocessor + + +class LiveInferencePipeline: + """End-to-end live inference from raw inputs. + + Handles: + - Feature extraction from raw video/audio/text + - Preprocessing and normalization + - Feature validation + - Model inference + - Result post-processing + """ + + def __init__( + self, + checkpoint_path: Path, + model_config: Dict[str, Any], + feature_spec: FeatureSpecification, + preprocessors: Optional[Dict[str, FeaturePreprocessor]] = None, + device: str = "auto", + ): + """Initialize live inference pipeline. + + Args: + checkpoint_path: Path to trained model checkpoint + model_config: Model configuration dict + feature_spec: Feature specification for validation + preprocessors: Optional preprocessors (normalization, PCA) + device: Device for inference ("auto", "cuda", "cpu") + """ + if device == "auto": + device = "cuda" if torch.cuda.is_available() else "cpu" + + self.device = torch.device(device) + self.feature_spec = feature_spec + self.preprocessors = preprocessors or {} + + # Load model + self.model = MultimodalDepressionModel(**model_config) + load_checkpoint(self.model, checkpoint_path, device=str(self.device)) + self.model.to(self.device).eval() + + # Feature extractors + resnet_pca = self.preprocessors.get("video_resnet_pca") if "video" in self.preprocessors else None + audio_normalizer = ( + self.preprocessors["audio"].normalizer + if "audio" in self.preprocessors else None + ) + + self.feature_extractor = LiveInferenceFeatureExtractor( + openface_pca=None, # OpenFace is not PCA-reduced + resnet_pca=resnet_pca, + audio_normalizer=audio_normalizer, + device=str(self.device), + ) + + # Validator + self.validator = FeatureValidator(feature_spec) + + def extract_video_features( + self, + video_path: Path | str, + openface_csv_path: Optional[Path | str] = None, + ) -> np.ndarray: + """Extract video features from OpenFace CSV. + + Args: + video_path: Path to video file (not used if CSV provided) + openface_csv_path: Path to pre-computed OpenFace CSV + + Returns: + Feature array [num_frames, feature_dim] + """ + if openface_csv_path: + return self.feature_extractor.openface_extractor.extract(openface_csv_path) + else: + raise ValueError( + "OpenFace extraction from raw video requires OpenFace binary. " + "Please provide pre-computed OpenFace CSV." + ) + + def extract_audio_features(self, audio_path: Path | str) -> np.ndarray: + """Extract audio features from file. + + Args: + audio_path: Path to audio file + + Returns: + Feature array [num_frames, feature_dim] + """ + return self.feature_extractor.extract_audio(audio_path)["mfcc"] + + def extract_text_features(self, text: str) -> np.ndarray: + """Extract text features from transcript. + + Args: + text: Transcript text + + Returns: + Feature array [num_chunks, feature_dim] + """ + return self.feature_extractor.extract_text(text) + + def _prepare_feature_tensor( + self, + features: Optional[np.ndarray], + expected_dim: int, + ) -> tuple[Optional[torch.Tensor], torch.Tensor]: + """Prepare feature tensor for model inference. + + Args: + features: Feature array or None + expected_dim: Expected feature dimension + + Returns: + Tuple of (feature_tensor, mask) + """ + if features is None: + return None, torch.zeros(1, 0, dtype=torch.bool, device=self.device) + + # Validate dimensions + if features.ndim != 2: + raise ValueError(f"Expected 2D features, got {features.ndim}D") + if features.shape[1] != expected_dim: + raise ValueError( + f"Feature dimension mismatch. " + f"Expected {expected_dim}, got {features.shape[1]}" + ) + + # Convert to tensor + tensor = torch.from_numpy(features).unsqueeze(0).to(self.device) # [1, time, dim] + mask = torch.ones(1, features.shape[0], dtype=torch.bool, device=self.device) + + return tensor, mask + + def predict_single( + self, + audio_path: Optional[Path | str] = None, + video_openface_csv: Optional[Path | str] = None, + transcript: Optional[str] = None, + verbose: bool = False, + ) -> Dict[str, Any]: + """Run inference on a single sample. + + Args: + audio_path: Path to audio file + video_openface_csv: Path to OpenFace CSV (video features) + transcript: Transcript text + verbose: Whether to print detailed output + + Returns: + Dictionary with predictions and metadata + """ + # Extract features + features = {} + try: + if audio_path: + features["audio"] = self.extract_audio_features(audio_path) + if verbose: + print(f" Audio features: {features['audio'].shape}") + + if video_openface_csv: + features["video"] = self.extract_video_features( + None, + video_openface_csv, + ) + if verbose: + print(f" Video features: {features['video'].shape}") + + if transcript: + features["text"] = self.extract_text_features(transcript) + if verbose: + print(f" Text features: {features['text'].shape}") + except Exception as e: + raise RuntimeError(f"Feature extraction failed: {e}") + + # Validate features + validation = self.validator.validate_batch(features) + if not validation.is_valid: + print(validation.report()) + raise ValueError("Feature validation failed") + + # Prepare tensors + audio_tensor, audio_mask = self._prepare_feature_tensor( + features.get("audio"), + self.feature_spec.audio_dim, + ) + video_tensor, video_mask = self._prepare_feature_tensor( + features.get("video"), + self.feature_spec.video_dim, + ) + text_tensor, text_mask = self._prepare_feature_tensor( + features.get("text"), + self.feature_spec.text_dim, + ) + + # Run inference + with torch.no_grad(): + output = self.model( + audio_tensor, + video_tensor, + text_tensor, + audio_mask, + video_mask, + text_mask, + ) + + # Post-process outputs + phq_total = float(output["phq_total"].cpu().numpy()[0]) + phq_questions = output["phq_questions"].cpu().numpy()[0].tolist() + classification = float(torch.sigmoid(output["classification"]).cpu().numpy()[0]) + confidence = float(output["confidence"].cpu().numpy()[0]) + + # Modality scores + modality_scores = {} + if "modality_scores" in output: + for key, val in output["modality_scores"].items(): + if isinstance(val, torch.Tensor): + modality_scores[key] = float(val.cpu().numpy()) + else: + modality_scores[key] = float(val) + + # Modality confidence + modality_confidence = {} + if "modality_confidence" in output: + conf_array = output["modality_confidence"].cpu().numpy()[0] + modality_confidence = { + "audio": float(conf_array[0]), + "video": float(conf_array[1]), + "text": float(conf_array[2]), + } + + entropy = float(output["entropy"].cpu().numpy()) + + # Binary classification (PHQ >= 10) + depression_binary = phq_total >= 10 + + return { + "phq_total": phq_total, + "phq_questions": phq_questions, + "classification_prob": classification, + "depression_likely": depression_binary, + "confidence": confidence, + "modality_scores": modality_scores, + "modality_confidence": modality_confidence, + "entropy": entropy, + "input_features": { + "audio_frames": features.get("audio").shape[0] if "audio" in features else 0, + "video_frames": features.get("video").shape[0] if "video" in features else 0, + "text_chunks": features.get("text").shape[0] if "text" in features else 0, + }, + } + + def predict_batch( + self, + samples: list[Dict[str, Any]], + verbose: bool = False, + ) -> list[Dict[str, Any]]: + """Run inference on multiple samples. + + Args: + samples: List of dicts with "audio_path", "video_openface_csv", "transcript" + verbose: Whether to print progress + + Returns: + List of prediction dicts + """ + results = [] + for i, sample in enumerate(samples): + if verbose: + print(f"Inference {i+1}/{len(samples)}...") + + result = self.predict_single( + audio_path=sample.get("audio_path"), + video_openface_csv=sample.get("video_openface_csv"), + transcript=sample.get("transcript"), + verbose=verbose, + ) + results.append(result) + + return results + + def predict_and_explain( + self, + audio_path: Optional[Path | str] = None, + video_openface_csv: Optional[Path | str] = None, + transcript: Optional[str] = None, + ) -> str: + """Run inference and generate human-readable report. + + Args: + audio_path: Path to audio file + video_openface_csv: Path to OpenFace CSV + transcript: Transcript text + + Returns: + Formatted report string + """ + result = self.predict_single( + audio_path=audio_path, + video_openface_csv=video_openface_csv, + transcript=transcript, + verbose=False, + ) + + lines = [] + lines.append("\n" + "=" * 70) + lines.append("MULTIMODAL BEHAVIORAL ASSESSMENT") + lines.append("=" * 70) + + # PHQ Score + lines.append(f"\nPHQ-9 Total Score: {result['phq_total']:.1f}/27") + if result["depression_likely"]: + lines.append(" → MODERATE TO SEVERE DEPRESSION (PHQ-9 ≥ 10)") + else: + lines.append(" → MINIMAL TO MILD DEPRESSION (PHQ-9 < 10)") + + # Individual questions + if result["phq_questions"]: + lines.append("\nPer-Question Scores:") + questions = [ + "Little interest/pleasure", + "Feeling down", + "Sleep problems", + "Fatigue/low energy", + "Appetite changes", + "Feeling bad about self", + "Concentration problems", + "Psychomotor changes", + ] + for i, (q, score) in enumerate(zip(questions, result["phq_questions"])): + lines.append(f" Q{i+1}: {q:30s} {score:.1f}") + + # Classification confidence + lines.append(f"\nClassification Confidence: {result['classification_prob']:.2%}") + lines.append(f"Prediction Confidence: {result['confidence']:.2%}") + + # Modality contributions + if result["modality_scores"]: + lines.append("\nModality Contributions to Prediction:") + for modality in ["audio", "video", "text"]: + if modality in result["modality_scores"]: + score = result["modality_scores"][modality] + conf = result["modality_confidence"].get(modality, 0.0) + lines.append(f" {modality.upper():8s}: weight={score:.2%}, confidence={conf:.2%}") + + lines.append(f"\nFusion Entropy: {result['entropy']:.4f}") + lines.append(f"Input Features:") + for feature_type, count in result["input_features"].items(): + lines.append(f" {feature_type}: {count}") + + lines.append("\n" + "=" * 70) + return "\n".join(lines) diff --git a/src/inference/model_io.py b/src/inference/model_io.py new file mode 100644 index 0000000000000000000000000000000000000000..b72a61c0c2b4a495bfd842df5f95f71f8ab68a6d --- /dev/null +++ b/src/inference/model_io.py @@ -0,0 +1,39 @@ +from pathlib import Path +from typing import Any, Dict +import torch + + +def save_checkpoint(model: torch.nn.Module, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + torch.save(model.state_dict(), path) + + +def load_checkpoint(model: torch.nn.Module, path: Path, device: str = "cpu") -> None: + state = torch.load(path, map_location=device) + # Support full training checkpoints that wrap the model state dict + if isinstance(state, dict) and "model_state_dict" in state: + state_dict = state["model_state_dict"] + else: + state_dict = state + # Load permissively to allow checkpoints saved from slightly different + # training wrappers or model definitions. Report missing/unexpected keys. + result = model.load_state_dict(state_dict, strict=False) + if result.missing_keys or result.unexpected_keys: + print(f"load_checkpoint: missing keys: {result.missing_keys}") + print(f"load_checkpoint: unexpected keys: {result.unexpected_keys}") + + +def safe_tensor(item: Any) -> Any: + if isinstance(item, torch.Tensor): + return item.detach().cpu().numpy() + return item + + +def sanitize_payload(payload: Dict[str, Any]) -> Dict[str, Any]: + sanitized: Dict[str, Any] = {} + for key, value in payload.items(): + if isinstance(value, float) and (value != value or value == float("inf") or value == float("-inf")): + sanitized[key] = 0.0 + else: + sanitized[key] = value + return sanitized diff --git a/src/inference/runtime_extractors.py b/src/inference/runtime_extractors.py new file mode 100644 index 0000000000000000000000000000000000000000..5868edc805014cdbc43813aa1d9f21df5fcb624c --- /dev/null +++ b/src/inference/runtime_extractors.py @@ -0,0 +1,451 @@ +"""Modular runtime feature extractors for live inference. + +These extractors automatically generate training-compatible features from raw +webcam/video/audio input without manual configuration. +""" + +from __future__ import annotations + +import csv +from pathlib import Path +from typing import Dict, List, Optional, Sequence +import numpy as np +import torch +import librosa +import torchvision.models as models +import torchvision.transforms as T +from transformers import Wav2Vec2FeatureExtractor, Wav2Vec2Model +from sentence_transformers import SentenceTransformer + +from src.features.sanitization import sanitize_array +from src.features.preprocessing import FeaturePreprocessor, PCATransform, FeatureNormalizer +from src.features.specs import AudioSpec, VideoSpec, TextSpec + + +class OpenFaceRuntimeExtractor: + """Extract OpenFace behavioral features from video frames or CSV. + + Supports: + - Direct frame-based extraction (requires OpenFace installed) + - CSV loading for pre-computed OpenFace features + """ + + EXPECTED_COLUMNS = [ + "AU01_r", "AU02_r", "AU04_r", "AU05_r", "AU06_r", "AU07_r", "AU09_r", + "AU10_r", "AU12_r", "AU14_r", "AU15_r", "AU17_r", "AU20_r", "AU23_r", + "AU25_r", "AU26_r", "AU45_r", # 17 AUs + "Gaze_angle_x", "Gaze_angle_y", "Gaze_angle_z", # 3 gaze angles + "Pose_Tx", "Pose_Ty", "Pose_Tz", # 3 head pose + ] + + def __init__(self, use_csv: bool = True): + """Initialize extractor. + + Args: + use_csv: If True, load from CSV. If False, use frame-based extraction. + """ + self.use_csv = use_csv + + def extract_from_csv(self, csv_path: Path) -> np.ndarray: + """Extract OpenFace features from CSV file. + + Args: + csv_path: Path to OpenFace CSV file + + Returns: + Features array [num_frames, num_features] + """ + if not csv_path.exists(): + raise FileNotFoundError(f"OpenFace CSV not found: {csv_path}") + + with open(csv_path, "r", encoding="utf-8") as f: + reader = csv.DictReader(f) + rows = list(reader) + + if not rows: + raise ValueError(f"OpenFace CSV is empty: {csv_path}") + + # Extract features + features_list = [] + for row in rows: + try: + # Extract AU features + au_features = [float(row.get(col, 0.0)) for col in self.EXPECTED_COLUMNS[:17]] + # Extract gaze + gaze_features = [float(row.get(col, 0.0)) for col in self.EXPECTED_COLUMNS[17:20]] + # Extract head pose + pose_features = [float(row.get(col, 0.0)) for col in self.EXPECTED_COLUMNS[20:23]] + + features_list.append(au_features + gaze_features + pose_features) + except (ValueError, KeyError) as e: + continue + + if not features_list: + raise ValueError(f"Could not extract valid features from {csv_path}") + + features = np.array(features_list, dtype=np.float32) + return sanitize_array(features) + + def extract(self, source: Path | str) -> np.ndarray: + """Extract OpenFace features from CSV or video frames.""" + source_path = Path(source) if isinstance(source, str) else source + + if source_path.suffix == ".csv": + return self.extract_from_csv(source_path) + else: + raise NotImplementedError( + "Frame-based OpenFace extraction requires additional dependencies. " + "Use CSV format for now." + ) + + +class ResNetRuntimeExtractor: + """Extract visual embeddings from video frames using ResNet50. + + Supports: + - Frame-by-frame extraction + - Temporal aggregation + - PCA reduction for consistency with training + """ + + def __init__( + self, + device: str = "auto", + pca_preprocessor: Optional[FeaturePreprocessor] = None, + ): + """Initialize extractor. + + Args: + device: Device to use ("auto", "cuda", "cpu") + pca_preprocessor: Optional PCA preprocessor for dimensionality reduction + """ + if device == "auto": + device = "cuda" if torch.cuda.is_available() else "cpu" + + self.device = torch.device(device) + self.pca_preprocessor = pca_preprocessor + + # Load pretrained ResNet50 + model = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V1) + self.backbone = torch.nn.Sequential(*list(model.children())[:-1]).to(self.device) + self.backbone.eval() + + # Image preprocessing + self.transform = T.Compose([ + T.Resize((224, 224)), + T.ToTensor(), + T.Normalize( + mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225], + ), + ]) + + def extract_from_frames(self, frames: Sequence[np.ndarray]) -> np.ndarray: + """Extract embeddings from video frames. + + Args: + frames: List of frames as np.ndarray [H, W, 3] + + Returns: + Embeddings array [num_frames, embedding_dim] + """ + if not frames: + raise ValueError("No frames provided") + + embeddings_list = [] + with torch.no_grad(): + for frame in frames: + if isinstance(frame, np.ndarray): + # Convert BGR to RGB if needed + if frame.shape[2] == 3: + if frame.max() > 1.0: # Likely 0-255 range + frame = frame.astype(np.float32) / 255.0 + frame_tensor = torch.from_numpy(frame).permute(2, 0, 1).float() + else: + frame_tensor = self.transform(T.ToPILImage()(frame)) + else: + frame_tensor = self.transform(frame) + + frame_tensor = frame_tensor.unsqueeze(0).to(self.device) + embedding = self.backbone(frame_tensor).squeeze() + embeddings_list.append(embedding.cpu().numpy()) + + embeddings = np.array(embeddings_list, dtype=np.float32) # [num_frames, 2048] + embeddings = sanitize_array(embeddings) + + # Apply PCA if available + if self.pca_preprocessor: + embeddings = self.pca_preprocessor.transform(embeddings) + + return embeddings + + +class AudioFeatureRuntimeExtractor: + """Extract audio features: MFCC + HuBERT. + + Produces training-compatible audio features. + """ + + def __init__( + self, + sample_rate: int = 16000, + mfcc_n_mfcc: int = 13, + use_hubert: bool = False, + normalizer: Optional[FeatureNormalizer] = None, + ): + """Initialize extractor. + + Args: + sample_rate: Audio sample rate + mfcc_n_mfcc: Number of MFCC coefficients + use_hubert: Whether to include HuBERT embeddings + normalizer: Optional normalizer for features + """ + self.sample_rate = sample_rate + self.mfcc_n_mfcc = mfcc_n_mfcc + self.use_hubert = use_hubert + self.normalizer = normalizer + + if use_hubert: + self._hubert_extractor = Wav2Vec2FeatureExtractor.from_pretrained( + "facebook/hubert-base-ls960" + ) + self._hubert_model = Wav2Vec2Model.from_pretrained("facebook/hubert-base-ls960") + self._hubert_model.eval() + if torch.cuda.is_available(): + self._hubert_model = self._hubert_model.cuda() + + def extract_mfcc(self, waveform: np.ndarray) -> np.ndarray: + """Extract MFCC features: 13 MFCC + 13 delta + 13 delta-delta. + + Args: + waveform: Audio waveform [num_samples] + + Returns: + MFCC features [num_frames, 39] + """ + waveform = np.asarray(waveform, dtype=np.float32) + if waveform.max() > 1.0: + waveform = waveform / 32768.0 # Convert from int16 + + # Normalize + waveform = librosa.util.normalize(waveform) + + # Extract MFCCs + mfcc = librosa.feature.mfcc(y=waveform, sr=self.sample_rate, n_mfcc=self.mfcc_n_mfcc) + delta = librosa.feature.delta(mfcc) + delta2 = librosa.feature.delta(mfcc, order=2) + + # Concatenate: [13 + 13 + 13 = 39] + features = np.vstack([mfcc, delta, delta2]).T # [num_frames, 39] + return sanitize_array(features.astype(np.float32)) + + def extract_hubert(self, waveform: np.ndarray) -> np.ndarray: + """Extract HuBERT embeddings. + + Args: + waveform: Audio waveform [num_samples] + + Returns: + HuBERT embeddings [num_frames, 768] + """ + if not self.use_hubert: + raise ValueError("HuBERT extraction not enabled") + + waveform = np.asarray(waveform, dtype=np.float32) + if waveform.max() > 1.0: + waveform = waveform / 32768.0 + + with torch.no_grad(): + inputs = self._hubert_extractor( + waveform, + sampling_rate=self.sample_rate, + return_tensors="pt", + padding=True, + ) + if torch.cuda.is_available(): + inputs = {k: v.cuda() for k, v in inputs.items()} + + outputs = self._hubert_model(**inputs) + embeddings = outputs.last_hidden_state[0].cpu().numpy() # [num_frames, 768] + + return sanitize_array(embeddings.astype(np.float32)) + + def extract(self, waveform: np.ndarray) -> Dict[str, np.ndarray]: + """Extract audio features. + + Args: + waveform: Audio waveform [num_samples] + + Returns: + Dict with 'mfcc' and optionally 'hubert' features + """ + features = {} + + # MFCC is always extracted + mfcc_features = self.extract_mfcc(waveform) + if self.normalizer: + mfcc_features = self.normalizer.transform(mfcc_features) + features["mfcc"] = mfcc_features + + # HuBERT if enabled + if self.use_hubert: + hubert_features = self.extract_hubert(waveform) + features["hubert"] = hubert_features + + return features + + +class TextFeatureRuntimeExtractor: + """Extract text embeddings using SentenceTransformer. + + Produces training-compatible text embeddings. + """ + + def __init__( + self, + model_name: str = "sentence-transformers/all-mpnet-base-v2", + device: str = "auto", + ): + """Initialize extractor. + + Args: + model_name: HuggingFace model name + device: Device to use ("auto", "cuda", "cpu") + """ + if device == "auto": + device = "cuda" if torch.cuda.is_available() else "cpu" + + self.device = device + self.model = SentenceTransformer(model_name, device=device) + + def encode_text(self, text: str) -> np.ndarray: + """Encode single text to embedding. + + Args: + text: Text string + + Returns: + Embedding [384] + """ + embedding = self.model.encode(text, convert_to_numpy=True) + return sanitize_array(embedding.astype(np.float32)) + + def encode_chunks(self, chunks: Sequence[str]) -> np.ndarray: + """Encode text chunks to embeddings. + + Args: + chunks: List of text chunks + + Returns: + Embeddings [num_chunks, 384] + """ + embeddings = self.model.encode( + list(chunks), + convert_to_numpy=True, + show_progress_bar=False, + ) + return sanitize_array(embeddings.astype(np.float32)) + + +class LiveInferenceFeatureExtractor: + """Unified runtime feature extraction for live inference. + + Automatically produces training-compatible features from raw inputs. + """ + + def __init__( + self, + openface_pca: Optional[PCATransform] = None, + resnet_pca: Optional[PCATransform] = None, + audio_normalizer: Optional[FeatureNormalizer] = None, + device: str = "auto", + ): + """Initialize extractor. + + Args: + openface_pca: Optional PCA for OpenFace features + resnet_pca: Optional PCA for ResNet embeddings + audio_normalizer: Optional normalizer for audio features + device: Device for model inference + """ + self.openface_extractor = OpenFaceRuntimeExtractor(use_csv=True) + self.resnet_extractor = ResNetRuntimeExtractor( + device=device, + pca_preprocessor=resnet_pca, + ) + self.audio_extractor = AudioFeatureRuntimeExtractor( + normalizer=audio_normalizer, + ) + self.text_extractor = TextFeatureRuntimeExtractor(device=device) + + def extract_video( + self, + source: Path | str, + use_openface_csv: bool = True, + ) -> Dict[str, np.ndarray]: + """Extract video features from file or CSV. + + Args: + source: Path to video file or OpenFace CSV + use_openface_csv: Whether source is OpenFace CSV + + Returns: + Dict with 'openface' and optionally 'resnet' features + """ + features = {} + + if use_openface_csv: + features["openface"] = self.openface_extractor.extract(source) + + return features + + def extract_audio(self, audio_path: Path | str) -> Dict[str, np.ndarray]: + """Extract audio features from file. + + Args: + audio_path: Path to audio file + + Returns: + Dict with 'mfcc' and optionally 'hubert' features + """ + waveform, sr = librosa.load(str(audio_path), sr=self.audio_extractor.sample_rate, mono=True) + return self.audio_extractor.extract(waveform) + + def extract_text( + self, + text: str, + chunk_size: int = 256, + ) -> np.ndarray: + """Extract text embeddings. + + Args: + text: Input text + chunk_size: Approximate chunk size for splitting + + Returns: + Embeddings [num_chunks, 384] + """ + # Simple sentence-based chunking + sentences = text.split(".") + chunks = [] + current_chunk = "" + + for sentence in sentences: + sentence = sentence.strip() + if not sentence: + continue + if len(current_chunk.split()) + len(sentence.split()) <= chunk_size: + current_chunk += " " + sentence if current_chunk else sentence + else: + if current_chunk: + chunks.append(current_chunk) + current_chunk = sentence + + if current_chunk: + chunks.append(current_chunk) + + if not chunks: + chunks = [text] + + return self.text_extractor.encode_chunks(chunks) diff --git a/src/inference/validation.py b/src/inference/validation.py new file mode 100644 index 0000000000000000000000000000000000000000..9496df6cbc8281ae7d62e96e81987921bc78819d --- /dev/null +++ b/src/inference/validation.py @@ -0,0 +1,355 @@ +"""Feature validation and train/inference consistency checking. + +This module ensures that training and inference pipelines produce identical +feature representations. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Dict, List, Optional, Tuple +import numpy as np +import torch +from dataclasses import dataclass + +from src.features.specs import FeatureSpecification, ModalitySpec +from src.features.preprocessing import FeaturePreprocessor, PCATransform, FeatureNormalizer + + +@dataclass +class FeatureValidationResult: + """Result of feature validation.""" + + is_valid: bool + errors: List[str] + warnings: List[str] + feature_dims: Dict[str, int] + + def report(self) -> str: + """Generate validation report.""" + lines = [] + lines.append("=" * 60) + lines.append("FEATURE VALIDATION REPORT") + lines.append("=" * 60) + + if self.is_valid: + lines.append("✓ All validations passed") + else: + lines.append("✗ Validation failed") + for error in self.errors: + lines.append(f" ERROR: {error}") + + if self.warnings: + lines.append("\nWarnings:") + for warning in self.warnings: + lines.append(f" WARNING: {warning}") + + lines.append("\nFeature Dimensions:") + for modality, dim in self.feature_dims.items(): + lines.append(f" {modality}: {dim}") + + lines.append("=" * 60) + return "\n".join(lines) + + +class FeatureValidator: + """Validate features match specification.""" + + def __init__(self, spec: FeatureSpecification): + """Initialize validator. + + Args: + spec: Feature specification to validate against + """ + self.spec = spec + + def validate_single_modality( + self, + features: np.ndarray, + modality: str, + ) -> Tuple[bool, List[str], List[str]]: + """Validate single modality features. + + Args: + features: Feature array + modality: Modality name ("audio", "video", "text") + + Returns: + Tuple of (is_valid, errors, warnings) + """ + errors = [] + warnings = [] + + try: + modal_spec = self.spec.get_spec(modality) + except ValueError: + errors.append(f"Unknown modality: {modality}") + return False, errors, warnings + + # Check dtype + if features.dtype != np.float32: + errors.append(f"{modality}: dtype is {features.dtype}, expected float32") + + # Check dimensionality + if modal_spec.temporal: + if features.ndim != 2: + errors.append(f"{modality}: expected 2D array [time, dim], got {features.ndim}D") + else: + if features.shape[1] != modal_spec.expected_dim: + errors.append( + f"{modality}: dimension mismatch. " + f"Expected {modal_spec.expected_dim}, got {features.shape[1]}" + ) + if features.shape[0] == 0: + errors.append(f"{modality}: zero-length temporal dimension") + else: + if features.ndim != 1: + errors.append(f"{modality}: expected 1D array, got {features.ndim}D") + if features.shape[0] != modal_spec.expected_dim: + errors.append( + f"{modality}: dimension mismatch. " + f"Expected {modal_spec.expected_dim}, got {features.shape[0]}" + ) + + # Check for NaN/Inf + if np.isnan(features).any(): + errors.append(f"{modality}: contains NaN values") + if np.isinf(features).any(): + errors.append(f"{modality}: contains Inf values") + + # Warnings for extreme values + if np.abs(features).max() > 100: + warnings.append(f"{modality}: contains very large values (max={np.abs(features).max():.2f})") + if np.abs(features[features != 0]).min() < 1e-6: + warnings.append(f"{modality}: contains very small non-zero values") + + is_valid = len(errors) == 0 + return is_valid, errors, warnings + + def validate_batch( + self, + features: Dict[str, np.ndarray], + ) -> FeatureValidationResult: + """Validate complete batch of features. + + Args: + features: Dict with "audio", "video", "text" keys + + Returns: + FeatureValidationResult with detailed report + """ + all_errors = [] + all_warnings = [] + feature_dims = {} + + for modality in ["audio", "video", "text"]: + if modality in features and features[modality] is not None: + array = features[modality] + is_valid, errors, warnings = self.validate_single_modality(array, modality) + all_errors.extend(errors) + all_warnings.extend(warnings) + + if array.ndim == 2: + feature_dims[modality] = array.shape[1] + elif array.ndim == 1: + feature_dims[modality] = array.shape[0] + + is_valid = len(all_errors) == 0 + return FeatureValidationResult( + is_valid=is_valid, + errors=all_errors, + warnings=all_warnings, + feature_dims=feature_dims, + ) + + +class TrainInferenceConsistencyChecker: + """Check that training and inference use consistent features.""" + + def __init__( + self, + spec: FeatureSpecification, + train_preprocessors: Optional[Dict[str, FeaturePreprocessor]] = None, + inference_preprocessors: Optional[Dict[str, FeaturePreprocessor]] = None, + ): + """Initialize consistency checker. + + Args: + spec: Feature specification + train_preprocessors: Preprocessors used during training + inference_preprocessors: Preprocessors used during inference + """ + self.spec = spec + self.train_preprocessors = train_preprocessors or {} + self.inference_preprocessors = inference_preprocessors or {} + self.validator = FeatureValidator(spec) + + def check_preprocessor_compatibility(self) -> Tuple[bool, List[str]]: + """Check that training and inference preprocessors are compatible. + + Returns: + Tuple of (is_compatible, issues) + """ + issues = [] + + for modality in ["audio", "video", "text"]: + train_prep = self.train_preprocessors.get(modality) + infer_prep = self.inference_preprocessors.get(modality) + + # Both should exist if either exists + if (train_prep is None) != (infer_prep is None): + issues.append( + f"{modality}: preprocessor exists in training but not in " + f"inference (or vice versa)" + ) + continue + + # If both exist, check compatibility + if train_prep and infer_prep: + if train_prep.normalizer_type != infer_prep.normalizer_type: + issues.append( + f"{modality}: different normalizer types " + f"(train={train_prep.normalizer_type}, " + f"infer={infer_prep.normalizer_type})" + ) + + if train_prep.apply_pca != infer_prep.apply_pca: + issues.append( + f"{modality}: PCA enabled in training but not in inference " + f"(or vice versa)" + ) + + if train_prep.apply_pca and infer_prep.apply_pca: + if train_prep.pca_components != infer_prep.pca_components: + issues.append( + f"{modality}: different PCA components " + f"(train={train_prep.pca_components}, " + f"infer={infer_prep.pca_components})" + ) + + is_compatible = len(issues) == 0 + return is_compatible, issues + + def validate_training_batch(self, batch: Dict[str, np.ndarray]) -> FeatureValidationResult: + """Validate training batch.""" + return self.validator.validate_batch(batch) + + def validate_inference_batch(self, batch: Dict[str, np.ndarray]) -> FeatureValidationResult: + """Validate inference batch.""" + return self.validator.validate_batch(batch) + + def generate_report(self) -> str: + """Generate comprehensive consistency report.""" + lines = [] + lines.append("\n" + "=" * 80) + lines.append("TRAIN/INFERENCE CONSISTENCY CHECK") + lines.append("=" * 80) + + # Check preprocessor compatibility + is_compatible, issues = self.check_preprocessor_compatibility() + if is_compatible: + lines.append("\n✓ Preprocessors are compatible between training and inference") + else: + lines.append("\n✗ Preprocessor compatibility issues found:") + for issue in issues: + lines.append(f" - {issue}") + + # Feature spec + lines.append(f"\nFeature Specification:") + lines.append(f" Audio: {self.spec.audio_dim}D") + lines.append(f" Video: {self.spec.video_dim}D") + lines.append(f" Text: {self.spec.text_dim}D") + + lines.append("\n" + "=" * 80) + return "\n".join(lines) + + +class FeatureCheckpoint: + """Save and restore preprocessors for exact reproducibility.""" + + def __init__(self, checkpoint_dir: Path): + """Initialize checkpoint manager. + + Args: + checkpoint_dir: Directory for storing checkpoints + """ + self.checkpoint_dir = Path(checkpoint_dir) + self.checkpoint_dir.mkdir(parents=True, exist_ok=True) + + def save_preprocessors( + self, + preprocessors: Dict[str, FeaturePreprocessor], + experiment_name: str, + ) -> None: + """Save preprocessors for later use. + + Args: + preprocessors: Dict of modality -> preprocessor + experiment_name: Name for this experiment + """ + exp_dir = self.checkpoint_dir / experiment_name / "preprocessors" + exp_dir.mkdir(parents=True, exist_ok=True) + + for modality, prep in preprocessors.items(): + prep_path = exp_dir / f"{modality}_preprocessor.pkl" + prep.save(prep_path) + + def load_preprocessors( + self, + experiment_name: str, + ) -> Dict[str, FeaturePreprocessor]: + """Load preprocessors from checkpoint. + + Args: + experiment_name: Experiment name + + Returns: + Dict of modality -> preprocessor + """ + exp_dir = self.checkpoint_dir / experiment_name / "preprocessors" + + if not exp_dir.exists(): + raise FileNotFoundError(f"Preprocessor checkpoint not found: {exp_dir}") + + preprocessors = {} + for modality in ["audio", "video", "text"]: + prep_path = exp_dir / f"{modality}_preprocessor.pkl" + if prep_path.exists(): + preprocessors[modality] = FeaturePreprocessor.load(prep_path) + + return preprocessors + + def save_feature_spec( + self, + spec: FeatureSpecification, + experiment_name: str, + ) -> None: + """Save feature specification.""" + import json + + exp_dir = self.checkpoint_dir / experiment_name + exp_dir.mkdir(parents=True, exist_ok=True) + + spec_path = exp_dir / "feature_spec.json" + with open(spec_path, "w") as f: + json.dump(spec.to_dict(), f, indent=2) + + def load_feature_spec(self, experiment_name: str) -> FeatureSpecification: + """Load feature specification.""" + import json + + spec_path = self.checkpoint_dir / experiment_name / "feature_spec.json" + if not spec_path.exists(): + raise FileNotFoundError(f"Feature spec not found: {spec_path}") + + with open(spec_path, "r") as f: + spec_dict = json.load(f) + + # Reconstruct FeatureSpecification from dict + from src.features.specs import ModalitySpec + + audio_spec = ModalitySpec.from_dict(spec_dict["audio"]) + video_spec = ModalitySpec.from_dict(spec_dict["video"]) + text_spec = ModalitySpec.from_dict(spec_dict["text"]) + + return FeatureSpecification(audio_spec, video_spec, text_spec) diff --git a/src/models/__init__.py b/src/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/models/__pycache__/__init__.cpython-312.pyc b/src/models/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3491b6675314729042a4e79a8fcbd9a8648b41e3 Binary files /dev/null and b/src/models/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/models/__pycache__/encoders.cpython-312.pyc b/src/models/__pycache__/encoders.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..49b5ffa6e93613f155bf94c12313ff19a9bafe66 Binary files /dev/null and b/src/models/__pycache__/encoders.cpython-312.pyc differ diff --git a/src/models/__pycache__/fusion.cpython-312.pyc b/src/models/__pycache__/fusion.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..317e1459e927bf4eac631245ffcac9ff7deb7f28 Binary files /dev/null and b/src/models/__pycache__/fusion.cpython-312.pyc differ diff --git a/src/models/__pycache__/heads.cpython-312.pyc b/src/models/__pycache__/heads.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..605ee238c2d0dac620c4c049a2426850ab4a9c12 Binary files /dev/null and b/src/models/__pycache__/heads.cpython-312.pyc differ diff --git a/src/models/__pycache__/multimodal_model.cpython-312.pyc b/src/models/__pycache__/multimodal_model.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe76de89292dd0bff94895a37540992d98cf5247 Binary files /dev/null and b/src/models/__pycache__/multimodal_model.cpython-312.pyc differ diff --git a/src/models/encoders.py b/src/models/encoders.py new file mode 100644 index 0000000000000000000000000000000000000000..987d85f0855e6797db639a445f2aa89e6bf0cc1c --- /dev/null +++ b/src/models/encoders.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import torch +import torch.nn as nn +from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence + + +class AttentionPooling(nn.Module): + def __init__(self, input_dim: int): + super().__init__() + self.query = nn.Parameter(torch.randn(input_dim)) + self.proj = nn.Linear(input_dim, input_dim) + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + attn = self.proj(x) + scores = torch.matmul(attn, self.query) + scores = scores.masked_fill(~mask, float("-1e9")) + weights = torch.softmax(scores, dim=1) + weights = weights.masked_fill(~mask, 0.0) + denom = weights.sum(dim=1, keepdim=True).clamp(min=1e-8) + weights = weights / denom + return (x * weights.unsqueeze(-1)).sum(dim=1) + + +class TemporalConvEncoder(nn.Module): + def __init__(self, input_dim: int, hidden_dim: int, kernel_size: int = 3, dropout: float = 0.1): + super().__init__() + hidden_dim = max(hidden_dim, 1) + self.input_norm = nn.LayerNorm(input_dim) + self.conv = nn.Sequential( + nn.Conv1d(in_channels=input_dim, out_channels=hidden_dim, kernel_size=kernel_size, padding=kernel_size // 2), + nn.GELU(), + nn.Dropout(dropout), + nn.Conv1d(hidden_dim, hidden_dim, kernel_size=kernel_size, padding=kernel_size // 2), + nn.BatchNorm1d(hidden_dim), + nn.GELU(), + ) + self.proj = nn.Linear(hidden_dim, hidden_dim) + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + x = self.input_norm(x) + x = x.transpose(1, 2) + out = self.conv(x).transpose(1, 2) + out = self.proj(out) + return out * mask.unsqueeze(-1).float() + + +class ConvGRUEncoder(nn.Module): + def __init__(self, input_dim: int, hidden_dim: int, num_layers: int = 1, dropout: float = 0.1): + super().__init__() + self.input_norm = nn.LayerNorm(input_dim) + self.conv = nn.Sequential( + nn.Conv1d(input_dim, hidden_dim, kernel_size=3, padding=1), + nn.GELU(), + nn.Dropout(dropout), + ) + self.gru = nn.GRU( + hidden_dim, + hidden_dim, + num_layers=num_layers, + batch_first=True, + bidirectional=True, + dropout=dropout if num_layers > 1 else 0.0, + ) + self.output_proj = nn.Linear(hidden_dim * 2, hidden_dim) + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + x = self.input_norm(x) + x = x.transpose(1, 2) + conv_out = self.conv(x).transpose(1, 2) + lengths = mask.sum(dim=1).cpu() + lengths_clamped = lengths.clamp(min=1) + packed = pack_padded_sequence(conv_out, lengths_clamped, batch_first=True, enforce_sorted=False) + packed_out, _ = self.gru(packed) + out, _ = pad_packed_sequence(packed_out, batch_first=True, total_length=conv_out.size(1)) + out = self.output_proj(out) + out = out * mask.unsqueeze(-1).float() + return out + +class BiGRUEncoder(nn.Module): + def __init__(self, input_dim: int, hidden_dim: int, num_layers: int = 1, dropout: float = 0.1): + super().__init__() + self.input_norm = nn.LayerNorm(input_dim) + self.gru = nn.GRU( + input_dim, + hidden_dim, + num_layers=num_layers, + batch_first=True, + bidirectional=True, + dropout=dropout if num_layers > 1 else 0.0, + ) + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + x = self.input_norm(x) + lengths = mask.sum(dim=1).cpu() + lengths_clamped = lengths.clamp(min=1) + packed = pack_padded_sequence(x, lengths_clamped, batch_first=True, enforce_sorted=False) + packed_out, _ = self.gru(packed) + out, _ = pad_packed_sequence(packed_out, batch_first=True, total_length=x.size(1)) + return out * mask.unsqueeze(-1).float() + + +class TransformerEncoder(nn.Module): + def __init__(self, input_dim: int, num_layers: int = 2, num_heads: int = 4, dropout: float = 0.1): + super().__init__() + self.input_norm = nn.LayerNorm(input_dim) + + self.embed_dim = ((input_dim + num_heads - 1) // num_heads) * num_heads + self.input_proj = nn.Linear(input_dim, self.embed_dim) if self.embed_dim != input_dim else nn.Identity() + + layer = nn.TransformerEncoderLayer( + d_model=self.embed_dim, + nhead=num_heads, + dim_feedforward=max(self.embed_dim * 2, 128), + dropout=dropout, + activation="gelu", + batch_first=True, + ) + self.transformer = nn.TransformerEncoder(layer, num_layers=num_layers) + self.output_proj = nn.Linear(self.embed_dim, input_dim) if self.embed_dim != input_dim else nn.Identity() + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + x = self.input_norm(x) + x = self.input_proj(x) + src_key_padding_mask = ~mask + out = self.transformer(x, src_key_padding_mask=src_key_padding_mask) + out = self.output_proj(out) + return out * mask.unsqueeze(-1).float() diff --git a/src/models/fusion.py b/src/models/fusion.py new file mode 100644 index 0000000000000000000000000000000000000000..394458b999b9d425a680059532585b931866812e --- /dev/null +++ b/src/models/fusion.py @@ -0,0 +1,356 @@ +from __future__ import annotations + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class CrossModalAttention(nn.Module): + """Cross-modal attention mechanism for multimodal fusion. + + Allows each modality to attend to other modalities for better integration. + """ + def __init__(self, embed_dim: int, num_heads: int = 4, dropout: float = 0.1): + super().__init__() + self.attention = nn.MultiheadAttention( + embed_dim=embed_dim, + num_heads=num_heads, + dropout=dropout, + batch_first=True, + ) + self.proj = nn.Sequential( + nn.Linear(embed_dim, embed_dim), + nn.GELU(), + nn.Dropout(dropout), + nn.LayerNorm(embed_dim), + ) + + def forward( + self, + modality_tokens: torch.Tensor, + key_padding_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + """Apply cross-modal attention. + + Args: + modality_tokens: [batch_size, num_modalities, embed_dim] + key_padding_mask: [batch_size, num_modalities] boolean mask + + Returns: + Attended tokens [batch_size, num_modalities, embed_dim] + """ + attended, _ = self.attention( + modality_tokens, + modality_tokens, + modality_tokens, + key_padding_mask=key_padding_mask, + ) + return self.proj(attended + modality_tokens) + + +class ModalityGating(nn.Module): + """Learnable modality gating without fixed floor constraints. + + Learns to weight each modality dynamically while respecting presence masks. + """ + def __init__( + self, + feature_dim: int, + hidden_dim: int = 256, + dropout: float = 0.1, + entropy_weight: float = 0.1, + ): + super().__init__() + self.entropy_weight = entropy_weight + + # Per-modality gating networks + self.modality_heads = nn.ModuleList( + [ + nn.Sequential( + nn.Linear(feature_dim, hidden_dim), + nn.GELU(), + nn.Dropout(dropout), + nn.Linear(hidden_dim, 1), + ) + for _ in range(3) + ] + ) + + def forward( + self, + modality_reprs: list[torch.Tensor], + modality_present: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Compute gating weights for modalities. + + Args: + modality_reprs: List of [batch_size, embed_dim] tensors for each modality + modality_present: [batch_size, 3] boolean mask of which modalities are present + + Returns: + Tuple of: + - weights: [batch_size, 3] normalized weights + - entropy: scalar entropy regularization loss + """ + # Compute logits for each modality + logits = torch.cat( + [head(mod) for head, mod in zip(self.modality_heads, modality_reprs)], + dim=-1, # [batch_size, 3] + ) + + # Apply presence masking before softmax + if modality_present is not None: + # Mask out absent modalities + logits = logits.masked_fill(~modality_present, float("-1e9")) + + # Compute weights via softmax + weights = F.softmax(logits, dim=-1) # [batch_size, 3] + + # Soft gate floor: ensure every *present* modality gets at least min_w weight. + # This prevents modality collapse (e.g. audio gate → 0.99) during training. + # Text is encouraged toward higher values via the text_gate_weight loss term + # in the loss function — not through a hard architectural floor, which would + # limit the residual learning budget and reduce overall CCC. + min_w = 0.15 + if modality_present is not None: + present_f = modality_present.float() # [B, 3] + num_present = present_f.sum(dim=-1, keepdim=True).clamp(min=1.0) + + # Floor: present modalities each get at least min_w, absent ones get 0. + # Renormalize so weights still sum to 1. + floored = weights * present_f # zero out absent modalities + floor_values = min_w * present_f # minimum per present modality + # Total floor mass consumed by floor + floor_sum = floor_values.sum(dim=-1, keepdim=True) # min_w * num_present + # Remaining mass to distribute via learned weights above the floor + remaining = (1.0 - floor_sum).clamp(min=0.0) + # Residual learned weights (above floor), re-normalised + residual_weights = (floored - floor_values).clamp(min=0.0) + residual_sum = residual_weights.sum(dim=-1, keepdim=True).clamp(min=1e-8) + residual_norm = residual_weights / residual_sum * remaining + weights = floor_values + residual_norm + + # Fallback: if all modalities absent, use uniform + no_present_rows = (present_f.sum(dim=-1) < 1e-6) + if no_present_rows.any(): + fallback = torch.ones_like(weights) / 3.0 + weights[no_present_rows] = fallback[no_present_rows] + else: + # No presence mask — apply floor uniformly across all 3 + floor_values = torch.full_like(weights, min_w) + remaining = 1.0 - min_w * 3 + residual = (weights - floor_values).clamp(min=0.0) + residual_sum = residual.sum(dim=-1, keepdim=True).clamp(min=1e-8) + residual_norm = residual / residual_sum * remaining + weights = floor_values + residual_norm + + # Compute entropy (max = log(3)≈1.099 for uniform distribution) + entropy = -(weights * torch.log(weights + 1e-8)).sum(dim=-1).mean() + + return weights, entropy + + +class ModalityConfidence(nn.Module): + """Per-modality confidence estimation for uncertainty awareness.""" + + def __init__(self, embed_dim: int, hidden_dim: int = 128, dropout: float = 0.1): + super().__init__() + self.audio_confidence = nn.Sequential( + nn.Linear(embed_dim, hidden_dim), + nn.GELU(), + nn.Dropout(dropout), + nn.Linear(hidden_dim, 1), + nn.Sigmoid(), + ) + self.video_confidence = nn.Sequential( + nn.Linear(embed_dim, hidden_dim), + nn.GELU(), + nn.Dropout(dropout), + nn.Linear(hidden_dim, 1), + nn.Sigmoid(), + ) + self.text_confidence = nn.Sequential( + nn.Linear(embed_dim, hidden_dim), + nn.GELU(), + nn.Dropout(dropout), + nn.Linear(hidden_dim, 1), + nn.Sigmoid(), + ) + + def forward( + self, + audio_repr: torch.Tensor, + video_repr: torch.Tensor, + text_repr: torch.Tensor, + ) -> torch.Tensor: + """Compute confidence scores for each modality. + + Returns: + Confidence scores [batch_size, 3] + """ + audio_conf = self.audio_confidence(audio_repr).squeeze(-1) # [batch_size] + video_conf = self.video_confidence(video_repr).squeeze(-1) + text_conf = self.text_confidence(text_repr).squeeze(-1) + + return torch.stack([audio_conf, video_conf, text_conf], dim=-1) # [batch_size, 3] + + +class MultimodalFusion(nn.Module): + """Enhanced multimodal fusion with entropy regularization and confidence scores. + + Supports multiple fusion strategies: + - early: Concatenate all modalities before fusion + - late: Gate modalities then concatenate with individual representations + - hybrid: Use cross-modal attention then gating (default, best for small datasets) + """ + + def __init__( + self, + embed_dim: int, + fusion_dim: int, + mode: str = "hybrid", + dropout: float = 0.2, + num_attention_heads: int = 4, + entropy_weight: float = 0.1, + ): + """Initialize multimodal fusion. + + Args: + embed_dim: Individual modality embedding dimension + fusion_dim: Fusion output dimension + mode: Fusion strategy ('early', 'late', 'hybrid') + dropout: Dropout rate + num_attention_heads: Number of attention heads + entropy_weight: Weight for entropy regularization (encourages diversity) + """ + super().__init__() + self.mode = mode + self.entropy_weight = entropy_weight + + # Gating mechanism (no min_gate floor - entropy regularization handles balance) + self.gating = ModalityGating( + feature_dim=embed_dim, + hidden_dim=fusion_dim, + dropout=dropout, + entropy_weight=entropy_weight, + ) + + # Cross-modal attention (optional, helps with hybrid mode) + self.cross_attention = CrossModalAttention( + embed_dim, + num_heads=num_attention_heads, + dropout=dropout, + ) + + # Confidence scoring + self.confidence = ModalityConfidence(embed_dim, hidden_dim=fusion_dim, dropout=dropout) + + # Determine fusion projection input dimension + if mode == "early": + projection_input_dim = embed_dim * 3 + else: # late or hybrid + projection_input_dim = embed_dim * 4 # gated_fusion + 3 individual reps + + # Fusion projection + self.fusion_projection = nn.Sequential( + nn.Linear(projection_input_dim, fusion_dim), + nn.GELU(), + nn.Dropout(dropout), + nn.LayerNorm(fusion_dim), + ) + + # Final fusion head + self.fusion_head = nn.Sequential( + nn.Linear(fusion_dim, fusion_dim), + nn.GELU(), + nn.Dropout(dropout), + nn.LayerNorm(fusion_dim), + ) + + def forward( + self, + audio_repr: torch.Tensor, + video_repr: torch.Tensor, + text_repr: torch.Tensor, + modality_present: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, dict[str, torch.Tensor], torch.Tensor, torch.Tensor]: + """Perform multimodal fusion. + + Args: + audio_repr: Audio representation [batch_size, embed_dim] + video_repr: Video representation [batch_size, embed_dim] + text_repr: Text representation [batch_size, embed_dim] + modality_present: [batch_size, 3] boolean mask of present modalities + + Returns: + Tuple of: + - fused: Fused representation [batch_size, fusion_dim] + - modality_scores: Dict with per-modality scores + - entropy: Entropy regularization loss + - confidence: Per-modality confidence [batch_size, 3] + """ + modality_reprs = [audio_repr, video_repr, text_repr] + + # Compute gating weights with entropy regularization + weights, entropy = self.gating(modality_reprs, modality_present) + + # Compute per-modality confidence scores + confidence = self.confidence(audio_repr, video_repr, text_repr) + + # Apply confidence-aware gating (emphasize confident modalities) + weighted_confidence = confidence * weights + confidence_sum = weighted_confidence.sum(dim=-1, keepdim=True).clamp(min=1e-8) + confidence_aware_weights = weighted_confidence / confidence_sum + + modality_scores = { + "audio": weights[:, 0].detach(), + "video": weights[:, 1].detach(), + "text": weights[:, 2].detach(), + "audio_confidence": confidence[:, 0].detach(), + "video_confidence": confidence[:, 1].detach(), + "text_confidence": confidence[:, 2].detach(), + # Non-detached weights so gate_balance_loss can backprop + "raw_weights": weights, + } + + # Apply fusion strategy + if self.mode == "early": + # Early fusion: just concatenate + fused_input = torch.cat(modality_reprs, dim=-1) + fused = self.fusion_projection(fused_input) + + elif self.mode == "late": + # Late fusion: gate then concatenate + gated = ( + audio_repr * weights[:, 0:1] + + video_repr * weights[:, 1:2] + + text_repr * weights[:, 2:3] + ) + fused_input = torch.cat([gated, audio_repr, video_repr, text_repr], dim=-1) + fused = self.fusion_projection(fused_input) + + else: # hybrid (default) + # Hybrid: cross-modal attention + gating + # Stack modalities for attention + tokens = torch.stack(modality_reprs, dim=1) # [batch_size, 3, embed_dim] + + # Create attention mask if modalities are missing + key_padding_mask = None + if modality_present is not None: + key_padding_mask = ~modality_present # [batch_size, 3] + + # Apply cross-modal attention + attended = self.cross_attention(tokens, key_padding_mask=key_padding_mask) + + # Apply gating weights to attended tokens + gated = attended * weights.unsqueeze(-1) + gated_fusion = gated.sum(dim=1) # [batch_size, embed_dim] + + # Concatenate gated fusion with individual representations + fused_input = torch.cat([gated_fusion, audio_repr, video_repr, text_repr], dim=-1) + fused = self.fusion_projection(fused_input) + + # Apply final fusion head + fused = self.fusion_head(fused) + + return fused, modality_scores, entropy, confidence \ No newline at end of file diff --git a/src/models/heads.py b/src/models/heads.py new file mode 100644 index 0000000000000000000000000000000000000000..128aa3a258ed092431cc553013ef2ed8aec1e8a3 --- /dev/null +++ b/src/models/heads.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import torch +import torch.nn as nn + + +class RegressionHead(nn.Module): + def __init__(self, input_dim: int): + super().__init__() + self.head = nn.Sequential( + nn.LayerNorm(input_dim), + nn.Dropout(0.2), + nn.Linear(input_dim, 1), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.head(x).squeeze(-1) + + +class QuestionHead(nn.Module): + def __init__(self, input_dim: int, num_questions: int = 8): + super().__init__() + self.head = nn.Sequential( + nn.LayerNorm(input_dim), + nn.Dropout(0.2), + nn.Linear(input_dim, num_questions), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.head(x) + + +class ClassificationHead(nn.Module): + def __init__(self, input_dim: int): + super().__init__() + self.head = nn.Sequential( + nn.LayerNorm(input_dim), + nn.Dropout(0.2), + nn.Linear(input_dim, 1), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.head(x).squeeze(-1) + + +class ConfidenceHead(nn.Module): + def __init__(self, input_dim: int): + super().__init__() + self.head = nn.Sequential( + nn.LayerNorm(input_dim), + nn.Dropout(0.2), + nn.Linear(input_dim, 1), + nn.Sigmoid(), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.head(x).squeeze(-1) diff --git a/src/models/multimodal_model.py b/src/models/multimodal_model.py new file mode 100644 index 0000000000000000000000000000000000000000..74621ef96b2f1945f6e9bc93e60ec5447c94cb13 --- /dev/null +++ b/src/models/multimodal_model.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import torch +import torch.nn as nn +from typing import Optional + +from src.models.encoders import ( + AttentionPooling, + BiGRUEncoder, + ConvGRUEncoder, + TemporalConvEncoder, + TransformerEncoder, +) +from src.models.fusion import MultimodalFusion +from src.models.heads import ClassificationHead, ConfidenceHead, QuestionHead, RegressionHead + + +class MultimodalDepressionModel(nn.Module): + def __init__( + self, + audio_dim: int = 512, + video_dim: int = 512, + text_dim: int = 384, + fusion_dim: int = 256, + num_questions: int = 8, + encoder_type: str = "transformer", + pooling_type: str = "attention", + num_heads: int = 4, + num_layers: int = 2, + dropout: float = 0.2, + fusion_mode: str = "hybrid", + modality_dropout: float = 0.1, + ): + super().__init__() + self.encoder_type = encoder_type + self.pooling_type = pooling_type + self.audio_input_dim = audio_dim + self.video_input_dim = video_dim + self.text_input_dim = text_dim + + self.audio_encoder = self._build_encoder(audio_dim, fusion_dim, dropout, num_layers, num_heads) + self.video_encoder = self._build_encoder(video_dim, fusion_dim, dropout, num_layers, num_heads) + self.text_encoder = self._build_encoder(text_dim, fusion_dim, dropout, num_layers, num_heads) + + self.audio_projection = self._build_projection(audio_dim, fusion_dim) + self.video_projection = self._build_projection(video_dim, fusion_dim) + self.text_projection = self._build_projection(text_dim, fusion_dim) + + self.audio_missing = nn.Parameter(torch.zeros(fusion_dim)) + self.video_missing = nn.Parameter(torch.zeros(fusion_dim)) + self.text_missing = nn.Parameter(torch.zeros(fusion_dim)) + + # Persistent attention poolers — one per modality so weights ARE trained. + # (previously created fresh each forward pass → random, never trained → CCC gap) + if self.pooling_type == "attention": + self.audio_pooler = AttentionPooling(fusion_dim) + self.video_pooler = AttentionPooling(fusion_dim) + self.text_pooler = AttentionPooling(fusion_dim) + + self.fusion = MultimodalFusion( + embed_dim=fusion_dim, + fusion_dim=fusion_dim, + mode=fusion_mode, + dropout=dropout, + ) + self.phq_head = RegressionHead(fusion_dim) + self.question_head = QuestionHead(fusion_dim, num_questions) + self.classification_head = ClassificationHead(fusion_dim) + self.confidence_head = ConfidenceHead(fusion_dim) + + self.modality_dropout = modality_dropout + + def _build_encoder( + self, + input_dim: int, + hidden_dim: int, + dropout: float, + num_layers: int, + num_heads: int, + ) -> nn.Module: + if self.encoder_type == "conv": + return TemporalConvEncoder(input_dim, hidden_dim, dropout=dropout) + if self.encoder_type == "conv_gru": + return ConvGRUEncoder(input_dim, hidden_dim, num_layers=num_layers, dropout=dropout) + if self.encoder_type == "bigru": + return BiGRUEncoder(input_dim, hidden_dim, num_layers=num_layers, dropout=dropout) + return TransformerEncoder(input_dim, num_layers=num_layers, num_heads=num_heads, dropout=dropout) + + def _build_projection(self, input_dim: int, hidden_dim: int) -> nn.Module: + if self.encoder_type == "bigru": + return nn.Linear(hidden_dim * 2, hidden_dim) + if self.encoder_type == "transformer": + return nn.Linear(input_dim, hidden_dim) + return nn.Linear(hidden_dim, hidden_dim) + + @staticmethod + def _mean_pool(x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + mask = mask.unsqueeze(-1).float() + summed = (x * mask).sum(dim=1) + length = mask.sum(dim=1).clamp(min=1.0) + return summed / length + + def _pooled_representation( + self, x: torch.Tensor, mask: torch.Tensor, pooler: Optional[nn.Module] = None + ) -> torch.Tensor: + if self.pooling_type == "attention" and pooler is not None: + return pooler(x, mask) + return self._mean_pool(x, mask) + + def _encode_modality( + self, + x: Optional[torch.Tensor], + mask: Optional[torch.Tensor], + encoder: nn.Module, + projector: nn.Module, + missing_token: nn.Parameter, + pooler: Optional[nn.Module] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if x is None or x.numel() == 0 or mask is None: + batch = x.shape[0] if x is not None else 1 + return missing_token.unsqueeze(0).expand(batch, -1), torch.zeros(batch, dtype=torch.bool, device=missing_token.device) + + mask = mask.to(dtype=torch.bool) + present = mask.any(dim=1) + encoded = encoder(x, mask) + pooled = self._pooled_representation(encoded, mask, pooler) + projected = projector(pooled) + + if not present.all(): + fallback = missing_token.unsqueeze(0).expand(projected.size(0), -1) + projected = projected * present.unsqueeze(-1).to(projected.dtype) + fallback * (~present).unsqueeze(-1).to(projected.dtype) + + return projected, present + + def _drop_modality( + self, + modality_repr: torch.Tensor, + present: torch.Tensor, + missing_token: nn.Parameter, + ) -> tuple[torch.Tensor, torch.Tensor]: + if not self.training or self.modality_dropout <= 0.0: + return modality_repr, present + + keep = torch.rand(modality_repr.size(0), device=modality_repr.device) > self.modality_dropout + keep = keep & present + if keep.all(): + return modality_repr, present + + fallback = missing_token.unsqueeze(0).expand_as(modality_repr) + modality_repr = modality_repr * keep.unsqueeze(-1).to(modality_repr.dtype) + fallback * (~keep).unsqueeze(-1).to(modality_repr.dtype) + return modality_repr, keep + + def forward( + self, + audio: Optional[torch.Tensor], + video: Optional[torch.Tensor], + text: Optional[torch.Tensor], + audio_mask: Optional[torch.Tensor], + video_mask: Optional[torch.Tensor], + text_mask: Optional[torch.Tensor], + ) -> dict[str, torch.Tensor]: + audio_repr, audio_present = self._encode_modality( + audio, + audio_mask, + self.audio_encoder, + self.audio_projection, + self.audio_missing, + pooler=getattr(self, "audio_pooler", None), + ) + video_repr, video_present = self._encode_modality( + video, + video_mask, + self.video_encoder, + self.video_projection, + self.video_missing, + pooler=getattr(self, "video_pooler", None), + ) + text_repr, text_present = self._encode_modality( + text, + text_mask, + self.text_encoder, + self.text_projection, + self.text_missing, + pooler=getattr(self, "text_pooler", None), + ) + + if self.training and self.modality_dropout > 0.0: + a_keep = torch.rand(audio_repr.size(0), device=audio_repr.device) > self.modality_dropout + v_keep = torch.rand(video_repr.size(0), device=video_repr.device) > self.modality_dropout + t_keep = torch.rand(text_repr.size(0), device=text_repr.device) > self.modality_dropout + + # If all are false, force one to be true among the ones that are actually present + a_keep = a_keep & audio_present + v_keep = v_keep & video_present + t_keep = t_keep & text_present + all_dropped = ~(a_keep | v_keep | t_keep) + + # For rows where all dropped, try text, then audio, then video + if all_dropped.any(): + t_keep = t_keep | (all_dropped & text_present) + all_dropped_still = all_dropped & ~text_present + a_keep = a_keep | (all_dropped_still & audio_present) + all_dropped_still = all_dropped_still & ~audio_present + v_keep = v_keep | (all_dropped_still & video_present) + + def apply_drop(repr_t, present_t, keep_t, missing_t): + if keep_t.all(): return repr_t, present_t + fallback = missing_t.unsqueeze(0).expand_as(repr_t) + repr_out = repr_t * keep_t.unsqueeze(-1).to(repr_t.dtype) + fallback * (~keep_t).unsqueeze(-1).to(repr_t.dtype) + return repr_out, keep_t + + audio_repr, audio_present = apply_drop(audio_repr, audio_present, a_keep, self.audio_missing) + video_repr, video_present = apply_drop(video_repr, video_present, v_keep, self.video_missing) + text_repr, text_present = apply_drop(text_repr, text_present, t_keep, self.text_missing) + + modality_present = torch.stack([audio_present, video_present, text_present], dim=1) + fused, modality_scores, entropy, confidence = self.fusion( + audio_repr, + video_repr, + text_repr, + modality_present=modality_present, + ) + + return { + "phq_total": self.phq_head(fused).squeeze(-1), + "phq_questions": self.question_head(fused), + "classification": self.classification_head(fused).squeeze(-1), + "confidence": self.confidence_head(fused).squeeze(-1), + "modality_scores": modality_scores, + "modality_confidence": confidence, + "fusion_entropy": entropy, + "entropy": entropy, + } diff --git a/src/models/temporal_modules.py b/src/models/temporal_modules.py new file mode 100644 index 0000000000000000000000000000000000000000..00bf4fb93c297ac1d68b2575c9b39e7e65b3dc6e --- /dev/null +++ b/src/models/temporal_modules.py @@ -0,0 +1,319 @@ +"""Improved temporal modeling architectures optimized for small datasets. + +These architectures prioritize: +- Generalization with limited data +- Stable convergence +- Parameter efficiency +- Temporal coherence +""" + +from __future__ import annotations + +import torch +import torch.nn as nn +from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence + + +class LightweightTemporalCNN(nn.Module): + """Lightweight temporal CNN for small datasets. + + Uses depthwise-separable convolutions to reduce parameters. + """ + + def __init__( + self, + input_dim: int, + hidden_dim: int, + num_layers: int = 2, + kernel_size: int = 3, + dropout: float = 0.2, + ): + super().__init__() + self.input_norm = nn.LayerNorm(input_dim) + + layers = [] + in_channels = input_dim + out_channels = hidden_dim + + for i in range(num_layers): + # Depthwise-separable convolution + layers.append(nn.Sequential( + # Depthwise: apply 1 filter per input channel + nn.Conv1d( + in_channels, + in_channels, + kernel_size=kernel_size, + padding=kernel_size // 2, + groups=in_channels, + bias=False, + ), + # Pointwise: 1x1 convolution + nn.Conv1d(in_channels, out_channels, kernel_size=1), + nn.GELU(), + nn.Dropout(dropout), + )) + in_channels = out_channels + + self.conv_stack = nn.Sequential(*layers) + self.output_proj = nn.Linear(hidden_dim, hidden_dim) + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + """Process temporal features. + + Args: + x: [batch_size, time_steps, input_dim] + mask: [batch_size, time_steps] + + Returns: + [batch_size, time_steps, hidden_dim] + """ + x = self.input_norm(x) + x = x.transpose(1, 2) # [batch, input_dim, time] + x = self.conv_stack(x) + x = x.transpose(1, 2) # [batch, time, hidden_dim] + x = self.output_proj(x) + return x * mask.unsqueeze(-1).float() + + +class StableGRUEncoder(nn.Module): + """Stable GRU encoder for small datasets. + + Uses layer normalization and careful initialization for better convergence. + """ + + def __init__( + self, + input_dim: int, + hidden_dim: int, + num_layers: int = 1, + dropout: float = 0.2, + bidirectional: bool = True, + ): + super().__init__() + self.input_norm = nn.LayerNorm(input_dim) + self.hidden_dim = hidden_dim + self.bidirectional = bidirectional + + self.gru = nn.GRU( + input_dim, + hidden_dim, + num_layers=num_layers, + batch_first=True, + bidirectional=bidirectional, + dropout=dropout if num_layers > 1 else 0.0, + ) + + # Projection for bidirectional output + if bidirectional: + self.output_proj = nn.Sequential( + nn.Linear(hidden_dim * 2, hidden_dim), + nn.LayerNorm(hidden_dim), + ) + else: + self.output_proj = nn.Identity() + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + """Process temporal features. + + Args: + x: [batch_size, time_steps, input_dim] + mask: [batch_size, time_steps] + + Returns: + [batch_size, time_steps, hidden_dim] + """ + x = self.input_norm(x) + lengths = mask.sum(dim=1).cpu() + lengths_clamped = lengths.clamp(min=1) + + # Pack for efficient RNN processing + packed = pack_padded_sequence(x, lengths_clamped, batch_first=True, enforce_sorted=False) + packed_out, _ = self.gru(packed) + out, _ = pad_packed_sequence(packed_out, batch_first=True, total_length=x.size(1)) + + # Project if bidirectional + out = self.output_proj(out) + + # Apply mask + return out * mask.unsqueeze(-1).float() + + +class HybridTemporalModule(nn.Module): + """Hybrid CNN+GRU for optimal temporal modeling. + + Combines CNN for local pattern detection with GRU for long-range dependencies. + Optimized for small datasets. + """ + + def __init__( + self, + input_dim: int, + hidden_dim: int, + dropout: float = 0.2, + ): + super().__init__() + self.input_norm = nn.LayerNorm(input_dim) + + # Light CNN for local patterns + self.conv = nn.Sequential( + nn.Conv1d(input_dim, hidden_dim, kernel_size=3, padding=1), + nn.GELU(), + nn.Dropout(dropout), + ) + + # GRU for temporal context + self.gru = nn.GRU( + hidden_dim, + hidden_dim, + num_layers=1, + batch_first=True, + bidirectional=True, + dropout=0.0, + ) + + # Output projection + self.proj = nn.Sequential( + nn.Linear(hidden_dim * 2, hidden_dim), + nn.LayerNorm(hidden_dim), + ) + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + """Process temporal features. + + Args: + x: [batch_size, time_steps, input_dim] + mask: [batch_size, time_steps] + + Returns: + [batch_size, time_steps, hidden_dim] + """ + x = self.input_norm(x) + + # CNN: local pattern extraction + x_t = x.transpose(1, 2) # [batch, input_dim, time] + x_conv = self.conv(x_t).transpose(1, 2) # [batch, time, hidden_dim] + + # GRU: temporal context + lengths = mask.sum(dim=1).cpu() + lengths_clamped = lengths.clamp(min=1) + packed = pack_padded_sequence(x_conv, lengths_clamped, batch_first=True, enforce_sorted=False) + packed_out, _ = self.gru(packed) + x_gru, _ = pad_packed_sequence(packed_out, batch_first=True, total_length=x.size(1)) + + # Project and mask + out = self.proj(x_gru) + return out * mask.unsqueeze(-1).float() + + +class AttentionPooling(nn.Module): + """Learnable attention pooling for temporal aggregation. + + Better than mean pooling for capturing important temporal elements. + """ + + def __init__(self, feature_dim: int, dropout: float = 0.1): + super().__init__() + self.query = nn.Parameter(torch.randn(feature_dim)) + self.proj = nn.Sequential( + nn.Linear(feature_dim, feature_dim), + nn.GELU(), + nn.Dropout(dropout), + ) + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + """Apply attention pooling. + + Args: + x: [batch_size, time_steps, feature_dim] + mask: [batch_size, time_steps] + + Returns: + Pooled output [batch_size, feature_dim] + """ + # Compute attention scores + attn = self.proj(x) # [batch, time, feature_dim] + scores = torch.matmul(attn, self.query) # [batch, time] + + # Apply mask + scores = scores.masked_fill(~mask, float("-1e9")) + + # Softmax + weights = torch.softmax(scores, dim=1) # [batch, time] + + # Weighted sum + return (x * weights.unsqueeze(-1)).sum(dim=1) # [batch, feature_dim] + + +class StatisticsPooling(nn.Module): + """Statistics-based temporal pooling (mean + std). + + Preserves information about feature variation which can be important + for behavioral signals. + """ + + def __init__(self, feature_dim: int): + super().__init__() + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + """Compute statistics over time. + + Args: + x: [batch_size, time_steps, feature_dim] + mask: [batch_size, time_steps] + + Returns: + Concatenated mean and std [batch_size, 2 * feature_dim] + """ + # Apply mask + x_masked = x * mask.unsqueeze(-1).float() + + # Compute mean + lengths = mask.sum(dim=1, keepdim=True).clamp(min=1) + mean = x_masked.sum(dim=1) / lengths # [batch, feature_dim] + + # Compute std + x_centered = (x - mean.unsqueeze(1)) * mask.unsqueeze(-1).float() + std = torch.sqrt((x_centered ** 2).sum(dim=1) / lengths + 1e-8) # [batch, feature_dim] + + # Concatenate + return torch.cat([mean, std], dim=-1) # [batch, 2 * feature_dim] + + +class TemporalAttentionAggregation(nn.Module): + """Multi-head attention for temporal aggregation. + + More expressive than single-head attention but still parameter-efficient. + """ + + def __init__( + self, + feature_dim: int, + num_heads: int = 4, + dropout: float = 0.1, + ): + super().__init__() + self.attention = nn.MultiheadAttention( + embed_dim=feature_dim, + num_heads=num_heads, + dropout=dropout, + batch_first=True, + ) + self.norm = nn.LayerNorm(feature_dim) + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + """Apply temporal self-attention. + + Args: + x: [batch_size, time_steps, feature_dim] + mask: [batch_size, time_steps] + + Returns: + Aggregated output [batch_size, feature_dim] + """ + # Self-attention + src_key_padding_mask = ~mask + attn_out, _ = self.attention(x, x, x, key_padding_mask=src_key_padding_mask) + attn_out = self.norm(attn_out + x) + + # Mean pooling + return (attn_out * mask.unsqueeze(-1).float()).sum(dim=1) / mask.sum(dim=1, keepdim=True).clamp(min=1) diff --git a/src/preprocessing/__init__.py b/src/preprocessing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/preprocessing/__pycache__/__init__.cpython-312.pyc b/src/preprocessing/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..01ee67971f9df4482f17b49bd2169e5894234b59 Binary files /dev/null and b/src/preprocessing/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/preprocessing/__pycache__/audio_preprocessor.cpython-312.pyc b/src/preprocessing/__pycache__/audio_preprocessor.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba11e2948158c8b8970fad980c9d205d5ca1333a Binary files /dev/null and b/src/preprocessing/__pycache__/audio_preprocessor.cpython-312.pyc differ diff --git a/src/preprocessing/__pycache__/chunker.cpython-312.pyc b/src/preprocessing/__pycache__/chunker.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1522a9b72cc10db0179d91a46f778b370104abeb Binary files /dev/null and b/src/preprocessing/__pycache__/chunker.cpython-312.pyc differ diff --git a/src/preprocessing/__pycache__/text_preprocessor.cpython-312.pyc b/src/preprocessing/__pycache__/text_preprocessor.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c6a3f0c16ffb4866289918a34fb864ff14a8e18e Binary files /dev/null and b/src/preprocessing/__pycache__/text_preprocessor.cpython-312.pyc differ diff --git a/src/preprocessing/__pycache__/vad.cpython-312.pyc b/src/preprocessing/__pycache__/vad.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..76d36fdb3b9401142cc0c70e203e21b067d1e48d Binary files /dev/null and b/src/preprocessing/__pycache__/vad.cpython-312.pyc differ diff --git a/src/preprocessing/__pycache__/video_preprocessor.cpython-312.pyc b/src/preprocessing/__pycache__/video_preprocessor.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..72ab3cae57818cd3d70b431e71eec2bb2a51d210 Binary files /dev/null and b/src/preprocessing/__pycache__/video_preprocessor.cpython-312.pyc differ diff --git a/src/preprocessing/audio_preprocessor.py b/src/preprocessing/audio_preprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..e1838ce4612c1ab2fe5515e6f4c4dbccd85994c1 --- /dev/null +++ b/src/preprocessing/audio_preprocessor.py @@ -0,0 +1,41 @@ +import asyncio +import math +import numpy as np +import librosa +from pathlib import Path +from typing import Sequence + +from src.preprocessing.vad import simple_energy_vad +from src.preprocessing.chunker import sliding_window +from src.features.sanitization import sanitize_array + + +class AudioPreprocessor: + """Preprocessing pipeline for raw audio waveforms.""" + + def __init__(self, sample_rate: int = 16000, frame_rate: int = 8, chunk_duration: float = 2.0, chunk_overlap: float = 0.5): + self.sample_rate = sample_rate + self.frame_rate = frame_rate + self.chunk_duration = chunk_duration + self.chunk_overlap = chunk_overlap + + async def load_audio(self, path: Path) -> np.ndarray: + waveform, sr = librosa.load(str(path), sr=self.sample_rate, mono=True) + return sanitize_array(waveform) + + def extract_chunks(self, waveform: np.ndarray) -> Sequence[np.ndarray]: + step = self.chunk_duration - self.chunk_overlap + hop_length = int(step * self.sample_rate) + window_length = int(self.chunk_duration * self.sample_rate) + return [segment for segment in sliding_window(waveform, window_length, hop_length) if segment.size == window_length] + + def apply_vad(self, chunks: Sequence[np.ndarray]) -> Sequence[np.ndarray]: + return [chunk for chunk in chunks if simple_energy_vad(chunk, self.sample_rate)] + + def preprocess(self, waveform: np.ndarray) -> np.ndarray: + waveform = sanitize_array(waveform) + chunks = self.extract_chunks(waveform) + chunks = self.apply_vad(chunks) + if not chunks: + return np.zeros((1, int(self.chunk_duration * self.sample_rate)), dtype=np.float32) + return np.stack(chunks, axis=0) diff --git a/src/preprocessing/chunker.py b/src/preprocessing/chunker.py new file mode 100644 index 0000000000000000000000000000000000000000..aaa6e2d92aeaca540fa45ba503aa97d2dfdb73f2 --- /dev/null +++ b/src/preprocessing/chunker.py @@ -0,0 +1,6 @@ +from typing import Sequence +import numpy as np + + +def sliding_window(array: np.ndarray, window_size: int, step_size: int) -> Sequence[np.ndarray]: + return [array[i : i + window_size] for i in range(0, len(array) - window_size + 1, step_size)] diff --git a/src/preprocessing/text_preprocessor.py b/src/preprocessing/text_preprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..81b7c0941de79442a5a981cb5d5280b5c56276c1 --- /dev/null +++ b/src/preprocessing/text_preprocessor.py @@ -0,0 +1,32 @@ +import re +from typing import List + + +class TextPreprocessor: + """Text preprocessing and chunking for transcript inputs.""" + + def normalize(self, text: str) -> str: + text = text.strip() + text = re.sub(r"\s+", " ", text) + return text + + def split_sentences(self, text: str) -> List[str]: + sentences = re.split(r"(?<=[.!?])\s+", text) + return [sentence.strip() for sentence in sentences if sentence.strip()] + + def chunk_text(self, text: str, max_tokens: int = 256) -> List[str]: + sentences = self.split_sentences(self.normalize(text)) + chunks: List[str] = [] + current: List[str] = [] + for sentence in sentences: + current.append(sentence) + if len(" ".join(current).split()) >= max_tokens: + chunks.append(" ".join(current)) + current = [] + if current: + chunks.append(" ".join(current)) + return chunks + + def preprocess(self, text: str) -> List[str]: + normalized = self.normalize(text) + return self.chunk_text(normalized) diff --git a/src/preprocessing/vad.py b/src/preprocessing/vad.py new file mode 100644 index 0000000000000000000000000000000000000000..dd319e4e0031ff7bdacacc4463ce9d13ec040a32 --- /dev/null +++ b/src/preprocessing/vad.py @@ -0,0 +1,8 @@ +import numpy as np + + +def simple_energy_vad(waveform: np.ndarray, sr: int, energy_threshold: float = 0.02) -> bool: + if waveform.size == 0: + return False + energy = np.mean(waveform ** 2) + return energy > energy_threshold diff --git a/src/preprocessing/video_preprocessor.py b/src/preprocessing/video_preprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..81faadcae5200e8623074f06bdbbe0c276ef6072 --- /dev/null +++ b/src/preprocessing/video_preprocessor.py @@ -0,0 +1,46 @@ +import asyncio +import subprocess +from pathlib import Path +from typing import List + +import numpy as np + + +class VideoPreprocessor: + """Async frame extraction and temporal window generation for video.""" + + def __init__(self, frame_rate: int = 8, temporal_window: int = 32): + self.frame_rate = frame_rate + self.temporal_window = temporal_window + + async def extract_frames(self, video_path: Path) -> List[np.ndarray]: + output_dir = Path(video_path.parent) / f"{video_path.stem}_frames" + output_dir.mkdir(parents=True, exist_ok=True) + command = [ + "ffmpeg", + "-y", + "-i", + str(video_path), + "-vf", + f"fps={self.frame_rate}", + str(output_dir / "%06d.jpg"), + ] + process = await asyncio.create_subprocess_exec(*command, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL) + await process.communicate() + paths = sorted(output_dir.glob("*.jpg")) + frames = [np.array(__import__("PIL.Image").Image.open(str(p)).convert("RGB")) for p in paths] + return frames + + def temporal_windows(self, features: np.ndarray) -> np.ndarray: + if features.ndim != 2: + raise ValueError("Video features must be a 2D array [frames, dim]") + num_frames = features.shape[0] + if num_frames == 0: + return np.zeros((1, self.temporal_window, features.shape[1]), dtype=np.float32) + stride = max(1, num_frames // self.temporal_window) + indices = list(range(0, num_frames, stride))[: self.temporal_window] + selected = features[indices] + if selected.shape[0] < self.temporal_window: + pad = np.zeros((self.temporal_window - selected.shape[0], features.shape[1]), dtype=np.float32) + selected = np.concatenate([selected, pad], axis=0) + return selected diff --git a/src/training/__init__.py b/src/training/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/training/__pycache__/__init__.cpython-312.pyc b/src/training/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ece7238f033a8a00e99aaed23a281667b2a20946 Binary files /dev/null and b/src/training/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/training/__pycache__/losses.cpython-312.pyc b/src/training/__pycache__/losses.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c9b49e0c52bb003a3c552481fc33b92d12be53d1 Binary files /dev/null and b/src/training/__pycache__/losses.cpython-312.pyc differ diff --git a/src/training/__pycache__/metrics.cpython-312.pyc b/src/training/__pycache__/metrics.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..41a58f85263827b0b255a2f47dad43e9fc21b6a2 Binary files /dev/null and b/src/training/__pycache__/metrics.cpython-312.pyc differ diff --git a/src/training/__pycache__/optimization.cpython-312.pyc b/src/training/__pycache__/optimization.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e4e13775321ac33e088cfc91017c3ae36d21b453 Binary files /dev/null and b/src/training/__pycache__/optimization.cpython-312.pyc differ diff --git a/src/training/__pycache__/trainer.cpython-312.pyc b/src/training/__pycache__/trainer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0280eff8a85e21ac1a4e99a3d0c30732533a036d Binary files /dev/null and b/src/training/__pycache__/trainer.cpython-312.pyc differ diff --git a/src/training/losses.py b/src/training/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..5502d63fa56df15b3f182f77947451fb377c3711 --- /dev/null +++ b/src/training/losses.py @@ -0,0 +1,214 @@ +import torch +import torch.nn as nn + + +class ConcordanceCorrelationCoefficientLoss(nn.Module): + """Concordance Correlation Coefficient loss for regression. + + Better than MSE for capturing correlation consistency. + """ + def __init__(self, epsilon: float = 1e-8): + super().__init__() + self.epsilon = epsilon + + def forward(self, predictions: torch.Tensor, targets: torch.Tensor) -> torch.Tensor: + mean_pred = predictions.mean(dim=0) + mean_true = targets.mean(dim=0) + var_pred = predictions.var(dim=0, unbiased=False) + var_true = targets.var(dim=0, unbiased=False) + covariance = ((predictions - mean_pred) * (targets - mean_true)).mean(dim=0) + ccc = (2.0 * covariance) / (var_pred + var_true + (mean_pred - mean_true).pow(2) + 1e-6) + return 1.0 - ccc.mean() + + +class ConfidenceAwareLoss(nn.Module): + """Confidence-aware regression loss. + + Uses per-sample confidence scores to weight regression targets. + Encourages the model to be confident when predictions are accurate. + """ + def __init__(self, epsilon: float = 1e-8): + super().__init__() + self.epsilon = epsilon + + def forward( + self, + predictions: torch.Tensor, + targets: torch.Tensor, + confidence: torch.Tensor, + ) -> torch.Tensor: + """Compute confidence-aware loss. + + Args: + predictions: Model predictions [batch_size] + targets: Ground truth targets [batch_size] + confidence: Confidence scores [batch_size] + + Returns: + Weighted loss with confidence regularization + """ + # Base MSE loss + mse = (predictions - targets).pow(2) + + # Weight by confidence (downweight uncertain predictions) + confident_loss = mse * confidence + + # Also penalize high confidence on bad predictions + error_magnitude = (predictions - targets).abs() + confidence_penalty = confidence * error_magnitude + + return (confident_loss + 0.1 * confidence_penalty).mean() + + +class EntropyRegularization(nn.Module): + """Entropy regularization to encourage multimodal diversity. + + Target is log(3) ≈ 1.099 — the maximum entropy for 3 modalities, meaning + truly balanced (uniform) gate weights. Penalises both collapse to one + modality and over-uniform (no learning). + """ + def __init__(self, target_entropy: float = 1.05): + super().__init__() + self.target_entropy = target_entropy + + def forward(self, entropy: torch.Tensor) -> torch.Tensor: + entropy_loss = (entropy - self.target_entropy).pow(2) + return entropy_loss + + +class GateBalanceLoss(nn.Module): + """Penalises deviation of gate weights from the uniform distribution (1/N). + + For 3 modalities the ideal unbiased weights are [1/3, 1/3, 1/3]. + This loss prevents any one modality from monopolising the fusion gate + which would make the model fragile during inference when modalities are + missing. + + Loss = mean over batch of ||w - 1/N||² + """ + + def __init__(self, num_modalities: int = 3): + super().__init__() + self.num_modalities = num_modalities + self.uniform = 1.0 / num_modalities + + def forward(self, gate_weights: torch.Tensor) -> torch.Tensor: + """ + Args: + gate_weights: [batch_size, num_modalities] softmax gate weights + + Returns: + Scalar balance loss + """ + deviation = (gate_weights - self.uniform).pow(2) + return deviation.mean() + + +class TextGatePreferenceLoss(nn.Module): + """Penalises when the text gate weight falls below a target floor. + + Encourages the model to rely more on linguistic features (text) which are + the most predictive modality for PHQ-8 depression detection. + Loss = mean(relu(target - w_text))^2 — only penalises *below* target. + """ + + def __init__(self, target: float = 0.40): + super().__init__() + self.target = target + + def forward(self, gate_weights: torch.Tensor) -> torch.Tensor: + """ + Args: + gate_weights: [batch_size, 3] — order: audio(0), video(1), text(2) + Returns: + Scalar loss (0 when text_gate >= target) + """ + text_gate = gate_weights[:, 2] # text is index 2 + shortfall = torch.relu(self.target - text_gate) + return shortfall.pow(2).mean() + + +class MultitaskLoss(nn.Module): + """Multitask loss combining regression, classification, question-level, + entropy regularisation, gate balance and confidence objectives.""" + + def __init__( + self, + regression_weight: float = 1.0, + classification_weight: float = 0.5, + question_weight: float = 1.0, + entropy_weight: float = 0.01, + ccc_weight: float = 0.2, + confidence_weight: float = 0.1, + gate_balance_weight: float = 0.5, + text_gate_weight: float = 0.0, + text_gate_target: float = 0.40, + ): + super().__init__() + self.regression_weight = regression_weight + self.classification_weight = classification_weight + self.question_weight = question_weight + self.entropy_weight = entropy_weight + self.ccc_weight = ccc_weight + self.confidence_weight = confidence_weight + self.gate_balance_weight = gate_balance_weight + self.text_gate_weight = text_gate_weight + + self.mse = nn.MSELoss() + self.bce = nn.BCEWithLogitsLoss() + self.ccc = ConcordanceCorrelationCoefficientLoss() + self.confidence_loss = ConfidenceAwareLoss() + self.entropy_reg = EntropyRegularization(target_entropy=1.05) + self.gate_balance = GateBalanceLoss(num_modalities=3) + self.text_gate_pref = TextGatePreferenceLoss(target=text_gate_target) + + def forward( + self, + phq_total_pred: torch.Tensor, + phq_total_target: torch.Tensor, + phq_questions_pred: torch.Tensor, + phq_questions_target: torch.Tensor, + classification_pred: torch.Tensor, + classification_target: torch.Tensor, + entropy_loss: torch.Tensor, + confidence_pred: torch.Tensor | None = None, + gate_weights: torch.Tensor | None = None, + ) -> torch.Tensor: + total_loss = 0.0 + + # CCC is the primary metric — give it the highest weight + if self.ccc_weight > 0.0: + total_loss += self.ccc_weight * self.ccc(phq_total_pred, phq_total_target) + + # Auxiliary regression (MSE) for stable gradient scale + regression_loss = self.mse(phq_total_pred, phq_total_target) + total_loss += self.regression_weight * regression_loss + + # Per-question regression + total_loss += self.question_weight * self.mse(phq_questions_pred, phq_questions_target) + + # Binary classification (depressed / not depressed) + total_loss += self.classification_weight * self.bce(classification_pred, classification_target) + + # Entropy regularisation — steer gates towards balanced attention + total_loss += self.entropy_weight * self.entropy_reg(entropy_loss) + + # Gate balance — penalise any gate collapsing to near-zero + if gate_weights is not None and self.gate_balance_weight > 0.0: + total_loss += self.gate_balance_weight * self.gate_balance(gate_weights) + + # Text gate preference — encourage text >= target floor + if gate_weights is not None and self.text_gate_weight > 0.0: + total_loss += self.text_gate_weight * self.text_gate_pref(gate_weights) + + # Confidence-aware loss + if confidence_pred is not None and self.confidence_weight > 0.0: + if confidence_pred.dim() > 1: + confidence_score = confidence_pred.mean(dim=-1) + else: + confidence_score = confidence_pred + confidence_loss = self.confidence_loss(phq_total_pred, phq_total_target, confidence_score) + total_loss += self.confidence_weight * confidence_loss + + return total_loss + diff --git a/src/training/metrics.py b/src/training/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..8b5032aacf4e9c48135b7ae3f61d0fe51b18ba0a --- /dev/null +++ b/src/training/metrics.py @@ -0,0 +1,29 @@ +import numpy as np +from sklearn.metrics import mean_squared_error, mean_absolute_error, f1_score, roc_auc_score + + +def concordance_correlation_coefficient(y_true: np.ndarray, y_pred: np.ndarray, epsilon: float = 1e-8) -> float: + true_mean = np.mean(y_true) + pred_mean = np.mean(y_pred) + covariance = np.mean((y_true - true_mean) * (y_pred - pred_mean)) + true_var = np.var(y_true) + pred_var = np.var(y_pred) + return (2 * covariance) / (true_var + pred_var + (true_mean - pred_mean) ** 2 + epsilon) + + +def regression_metrics(y_true: np.ndarray, y_pred: np.ndarray) -> dict[str, float]: + return { + "ccc": float(concordance_correlation_coefficient(y_true, y_pred)), + "rmse": float(np.sqrt(mean_squared_error(y_true, y_pred))), + "mae": float(mean_absolute_error(y_true, y_pred)), + } + + +def classification_metrics(y_true: np.ndarray, y_pred_probs: np.ndarray) -> dict[str, float]: + binary = (y_true >= 10).astype(int) + preds = (y_pred_probs >= 0.5).astype(int) + auc = roc_auc_score(binary, y_pred_probs) if len(np.unique(binary)) > 1 else 0.5 + return { + "f1": float(f1_score(binary, preds, zero_division=0)), + "roc_auc": float(auc), + } diff --git a/src/training/optimization.py b/src/training/optimization.py new file mode 100644 index 0000000000000000000000000000000000000000..b144215a56d679d6630d16ece93ac342d0758953 --- /dev/null +++ b/src/training/optimization.py @@ -0,0 +1,386 @@ +"""Training utilities for small dataset optimization. + +Implements advanced techniques: +- Exponential Moving Average (EMA) weights +- Checkpoint averaging +- Curriculum learning +- Gradient accumulation +- Warmup scheduling +""" + +from __future__ import annotations + +import torch +import torch.nn as nn +from pathlib import Path +from typing import Optional, List +import copy + + +class EMA(nn.Module): + """Exponential Moving Average for model weights. + + Maintains a running average of model weights for better generalization, + especially important for small datasets. + """ + + def __init__(self, model: nn.Module, decay: float = 0.999): + """Initialize EMA wrapper. + + Args: + model: Model to apply EMA to + decay: EMA decay rate (higher = slower updates) + """ + super().__init__() + self.decay = decay + self.model = model + self.shadow = copy.deepcopy(model) + self.shadow.eval() + + for param in self.shadow.parameters(): + param.requires_grad = False + + def update(self) -> None: + """Update shadow weights with EMA.""" + for model_param, shadow_param in zip(self.model.parameters(), self.shadow.parameters()): + shadow_param.data = self.decay * shadow_param.data + (1 - self.decay) * model_param.data.detach() + + def forward(self, *args, **kwargs): + """Forward pass through shadow model (during inference).""" + return self.shadow(*args, **kwargs) + + def swap(self) -> None: + """Swap model and shadow parameters for evaluation.""" + self.model, self.shadow = self.shadow, self.model + + def restore(self) -> None: + """Restore original model and shadow after evaluation.""" + self.swap() + + +class CheckpointAveraging: + """Average weights from multiple checkpoints for improved generalization.""" + + def __init__(self, num_checkpoints: int = 5): + """Initialize checkpoint averaging. + + Args: + num_checkpoints: Number of last checkpoints to average + """ + self.num_checkpoints = num_checkpoints + self.checkpoints: List[dict] = [] + + def add_checkpoint(self, state_dict: dict) -> None: + """Add checkpoint state dict. + + Args: + state_dict: Model state dict to add + """ + import copy + self.checkpoints.append(copy.deepcopy(state_dict)) + # Keep only recent checkpoints + if len(self.checkpoints) > self.num_checkpoints: + self.checkpoints.pop(0) + + def get_averaged_weights(self) -> dict: + """Compute average of checkpoint weights. + + Returns: + Averaged state dict + """ + if not self.checkpoints: + raise ValueError("No checkpoints to average") + + averaged = {} + num_checkpoints = len(self.checkpoints) + + # Get keys from first checkpoint + for key in self.checkpoints[0].keys(): + # Average weights + weight_sum = torch.zeros_like(self.checkpoints[0][key]) + for ckpt in self.checkpoints: + weight_sum += ckpt[key] + averaged[key] = weight_sum / num_checkpoints + + return averaged + + +class WarmupScheduler: + """Learning rate warmup + decay scheduler. + + Gradually increases LR during warmup phase, then decays. + Important for small datasets to prevent divergence. + """ + + def __init__( + self, + optimizer: torch.optim.Optimizer, + warmup_epochs: int = 5, + total_epochs: int = 50, + base_lr: float = 1e-4, + min_lr: float = 1e-6, + ): + """Initialize scheduler. + + Args: + optimizer: PyTorch optimizer + warmup_epochs: Number of epochs for warmup + total_epochs: Total number of training epochs + base_lr: Base learning rate + min_lr: Minimum learning rate + """ + self.optimizer = optimizer + self.warmup_epochs = warmup_epochs + self.total_epochs = total_epochs + self.base_lr = base_lr + self.min_lr = min_lr + self.epoch = 0 + + def step(self) -> None: + """Update learning rate for current epoch.""" + if self.epoch < self.warmup_epochs: + # Linear warmup + lr = self.base_lr * (self.epoch + 1) / self.warmup_epochs + else: + # Cosine annealing decay + progress = (self.epoch - self.warmup_epochs) / (self.total_epochs - self.warmup_epochs) + lr = self.min_lr + (self.base_lr - self.min_lr) * 0.5 * (1 + torch.cos(torch.tensor(progress * 3.14159)).item()) + + for param_group in self.optimizer.param_groups: + param_group["lr"] = lr + + self.epoch += 1 + + def get_lr(self) -> float: + """Get current learning rate.""" + return self.optimizer.param_groups[0]["lr"] + + +class CurriculumLearning: + """Curriculum learning scheduler for small datasets. + + Gradually increases task difficulty (e.g., more modality dropout). + Helps prevent overfitting on small datasets. + """ + + def __init__( + self, + initial_dropout_rate: float = 0.0, + final_dropout_rate: float = 0.2, + total_epochs: int = 50, + ): + """Initialize curriculum. + + Args: + initial_dropout_rate: Starting dropout rate (easy) + final_dropout_rate: Final dropout rate (hard) + total_epochs: Total training epochs + """ + self.initial_dropout = initial_dropout_rate + self.final_dropout = final_dropout_rate + self.total_epochs = total_epochs + self.epoch = 0 + + def get_current_difficulty(self) -> float: + """Get current difficulty level (0 to 1).""" + return self.epoch / self.total_epochs + + def get_modality_dropout_rate(self) -> float: + """Get modality dropout rate for current epoch.""" + difficulty = self.get_current_difficulty() + return self.initial_dropout + (self.final_dropout - self.initial_dropout) * difficulty + + def get_temporal_dropout_rate(self) -> float: + """Get temporal dropout rate for current epoch.""" + difficulty = self.get_current_difficulty() + # Start with less temporal dropout, increase over time + return difficulty * 0.15 + + def step(self) -> None: + """Advance to next epoch.""" + self.epoch += 1 + + +class GradientAccumulation: + """Gradient accumulation for large effective batch sizes on small datasets.""" + + def __init__(self, accumulation_steps: int = 4): + """Initialize gradient accumulation. + + Args: + accumulation_steps: Number of steps to accumulate gradients + """ + self.accumulation_steps = accumulation_steps + self.step_count = 0 + + def should_accumulate(self) -> bool: + """Check if should accumulate gradient.""" + return (self.step_count + 1) % self.accumulation_steps != 0 + + def should_step(self) -> bool: + """Check if should perform optimizer step.""" + return (self.step_count + 1) % self.accumulation_steps == 0 + + def step(self) -> None: + """Advance accumulation counter.""" + self.step_count += 1 + + def reset(self) -> None: + """Reset accumulation counter.""" + self.step_count = 0 + + +class MixupAugmentation: + """Mixup data augmentation for small datasets. + + Creates interpolated samples between training examples. + """ + + def __init__(self, alpha: float = 0.2): + """Initialize mixup. + + Args: + alpha: Beta distribution parameter for mixing + """ + self.alpha = alpha + + def __call__( + self, + features1: torch.Tensor, + features2: torch.Tensor, + target1: float, + target2: float, + ) -> tuple[torch.Tensor, float]: + """Apply mixup to two samples. + + Args: + features1: First feature tensor + features2: Second feature tensor + target1: First target value + target2: Second target value + + Returns: + Tuple of (mixed_features, mixed_target) + """ + lam = torch.distributions.Beta(self.alpha, self.alpha).sample() + mixed_features = lam * features1 + (1 - lam) * features2 + mixed_target = lam * target1 + (1 - lam) * target2 + return mixed_features, mixed_target + + +class ManifoldMixup: + """Manifold Mixup: apply mixup in hidden layer space. + + More effective than input-space mixup for small datasets. + """ + + def __init__(self, alpha: float = 0.2): + """Initialize manifold mixup. + + Args: + alpha: Beta distribution parameter + """ + self.alpha = alpha + + def __call__( + self, + hidden1: torch.Tensor, + hidden2: torch.Tensor, + target1: float, + target2: float, + ) -> tuple[torch.Tensor, float]: + """Apply mixup to hidden representations. + + Args: + hidden1: First hidden representation + hidden2: Second hidden representation + target1: First target value + target2: Second target value + + Returns: + Tuple of (mixed_hidden, mixed_target) + """ + lam = torch.distributions.Beta(self.alpha, self.alpha).sample() + mixed_hidden = lam * hidden1 + (1 - lam) * hidden2 + mixed_target = lam * target1 + (1 - lam) * target2 + return mixed_hidden, mixed_target + + +class StochasticDepth(nn.Module): + """Stochastic depth (drop path) for regularization. + + Randomly drops entire residual connections during training. + """ + + def __init__(self, drop_prob: float = 0.1): + """Initialize stochastic depth. + + Args: + drop_prob: Probability of dropping path + """ + super().__init__() + self.drop_prob = drop_prob + + def forward(self, x: torch.Tensor, residual: torch.Tensor) -> torch.Tensor: + """Apply stochastic depth. + + Args: + x: Main path output + residual: Residual connection + + Returns: + Output with stochastic depth applied + """ + if not self.training or self.drop_prob == 0: + return x + residual + + # Randomly drop path + keep_prob = 1 - self.drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) + random_tensor = torch.rand(shape, device=x.device) + random_tensor = (random_tensor < keep_prob).float() / keep_prob + return (x + residual) * random_tensor + + +class FocalLoss(nn.Module): + """Focal loss for class imbalance in binary classification. + + Downweights easy examples, focuses on hard ones. + """ + + def __init__(self, alpha: float = 0.25, gamma: float = 2.0): + """Initialize focal loss. + + Args: + alpha: Weighting factor in [0, 1] + gamma: Focusing parameter >= 0 + """ + super().__init__() + self.alpha = alpha + self.gamma = gamma + + def forward( + self, + predictions: torch.Tensor, + targets: torch.Tensor, + ) -> torch.Tensor: + """Compute focal loss. + + Args: + predictions: Logits [batch_size] + targets: Binary targets [batch_size] + + Returns: + Focal loss + """ + # Convert logits to probabilities + p = torch.sigmoid(predictions) + + # Compute cross entropy + ce = torch.nn.functional.binary_cross_entropy_with_logits(predictions, targets, reduction="none") + + # Compute focal loss + p_t = p * targets + (1 - p) * (1 - targets) + focal_loss = self.alpha * ((1 - p_t) ** self.gamma) * ce + + return focal_loss.mean() diff --git a/src/training/trainer.py b/src/training/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..0e50568ba601fd51c7a983e0f9bb62ba1e3d807b --- /dev/null +++ b/src/training/trainer.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import time +import torch +import torch.nn as nn +from pathlib import Path +from typing import Dict, Optional +from torch.utils.data import DataLoader + +from src.training.losses import MultitaskLoss +from src.training.metrics import regression_metrics +import logging + +logger = logging.getLogger(__name__) + + +class EarlyStopping: + def __init__(self, patience: int = 8, min_delta: float = 0.001): + self.patience = patience + self.min_delta = min_delta + self.best = float("inf") + self.counter = 0 + + def step(self, value: float) -> bool: + if value + self.min_delta < self.best: + self.best = value + self.counter = 0 + return False + self.counter += 1 + return self.counter >= self.patience + + +class Trainer: + def __init__( + self, + model: nn.Module, + train_loader: DataLoader, + val_loader: DataLoader, + optimizer: torch.optim.Optimizer, + scheduler: Optional[torch.optim.lr_scheduler._LRScheduler], + config: dict, + device: Optional[torch.device] = None, + ): + self.model = model + self.train_loader = train_loader + self.val_loader = val_loader + self.optimizer = optimizer + self.scheduler = scheduler + self.device = device or torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.model.to(self.device) + self.max_epochs = config.get("max_epochs", 50) + self.gradient_clip_norm = config.get("grad_clip_norm", 1.0) + self.use_amp = config.get("use_amp", True) + self.early_stopping = EarlyStopping(config.get("early_stopping_patience", 8), config.get("early_stopping_min_delta", 0.001)) + self.scaler = torch.cuda.amp.GradScaler(enabled=self.use_amp and self.device.type == "cuda") + self.loss_fn = MultitaskLoss( + regression_weight=config.get("regression_weight", 1.0), + classification_weight=config.get("classification_weight", 0.5), + question_weight=config.get("question_weight", 1.0), + entropy_weight=config.get("entropy_weight", 0.02), + ccc_weight=config.get("ccc_weight", 0.2), + confidence_weight=config.get("confidence_weight", 0.05), + gate_balance_weight=config.get("gate_balance_weight", 0.5), + text_gate_weight=config.get("text_gate_weight", 0.0), + text_gate_target=config.get("text_gate_target", 0.40), + ) + self.mixup_alpha = float(config.get("mixup_alpha", 0.0)) + self.checkpoint_dir = Path(config.get("checkpoint_dir", "checkpoints")) + self.checkpoint_dir.mkdir(parents=True, exist_ok=True) + # EMA for better generalization on small datasets + from src.training.optimization import EMA + self.ema = EMA(self.model, decay=0.999) + self.best_ccc = -float("inf") + + def train(self) -> Dict[str, float]: + history = { + "train_loss": [], + "val_loss": [], + "val_ccc": [], + "val_rmse": [], + "val_mae": [], + } + + logger.info("Starting training loop...") + best_path = self.checkpoint_dir / "best_model.pt" + best_ccc = -float("inf") + + for epoch in range(1, self.max_epochs + 1): + train_loss = self._train_epoch() + + # Use EMA weights for validation + self.ema.swap() + val_loss, val_metrics = self._validate_epoch() + self.ema.restore() + + val_ccc = val_metrics.get("ccc", val_metrics.get("total_ccc", 0.0)) + + history["train_loss"].append(train_loss) + history["val_loss"].append(val_loss) + history["val_rmse"].append(val_metrics.get("rmse", 0.0)) + history["val_mae"].append(val_metrics.get("mae", 0.0)) + history["val_ccc"].append(val_ccc) + + gate_str = "" + if "gate_audio" in val_metrics: + gate_str = f" | Gates: A:{val_metrics['gate_audio']:.2f} V:{val_metrics['gate_video']:.2f} T:{val_metrics['gate_text']:.2f}" + logger.info( + f"Epoch {epoch:03d}/{self.max_epochs} | " + f"Train Loss: {train_loss:.4f} | " + f"Val Loss: {val_loss:.4f} | " + f"Val CCC: {val_ccc:.4f} | " + f"Val RMSE: {val_metrics.get('rmse', val_metrics.get('total_rmse', 0.0)):.4f}{gate_str}" + ) + + if self.scheduler is not None: + self.scheduler.step(val_loss) + + if val_ccc > best_ccc: + best_ccc = val_ccc + self.best_ccc = best_ccc + logger.info(f" --> New best model saved with CCC: {val_ccc:.4f}") + # Save EMA weights as the best model + self.ema.swap() + torch.save(self.model.state_dict(), best_path) + self.ema.restore() + + if self.early_stopping.step(-val_ccc): # Early stopping based on CCC (negative because it expects min_delta to be lower) + logger.info(f"Early stopping triggered at epoch {epoch}") + break + + return {"best_ccc": best_ccc, **history} + + def _train_epoch(self) -> float: + self.model.train() + total_loss = 0.0 + for batch in self.train_loader: + self.optimizer.zero_grad() + loss = self._compute_batch_loss(batch) + self.scaler.scale(loss).backward() + self.scaler.unscale_(self.optimizer) + torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.gradient_clip_norm) + self.scaler.step(self.optimizer) + self.scaler.update() + + # Update EMA weights + self.ema.update() + + total_loss += float(loss.detach().cpu()) + return total_loss / max(1, len(self.train_loader)) + + def _validate_epoch(self) -> tuple[float, Dict[str, float]]: + self.model.eval() + total_loss = 0.0 + y_true = [] + y_pred = [] + all_modality_scores = [] + with torch.no_grad(): + for batch in self.val_loader: + output = self._forward(batch) + ms_val = output.get("modality_scores") + gate_w_val = None + if ms_val is not None and isinstance(ms_val, dict): + raw_v = ms_val.get("raw_weights") + gate_w_val = raw_v if raw_v is not None else torch.stack( + [ms_val["audio"], ms_val["video"], ms_val["text"]], dim=1 + ).to(self.device) + loss = self.loss_fn( + output["phq_total"], + batch["phq_total"].to(self.device), + output["phq_questions"], + batch["phq_questions"].to(self.device), + output["classification"], + batch["classification"].to(self.device), + output["entropy"], + confidence_pred=output.get("modality_confidence"), + gate_weights=gate_w_val, + ) + total_loss += float(loss.cpu()) + y_true.extend(batch["phq_total"].tolist()) + y_pred.extend(output["phq_total"].cpu().tolist()) + if "modality_scores" in output and output["modality_scores"] is not None: + ms = output["modality_scores"] + if isinstance(ms, dict): + gate_tensor = torch.stack([ms["audio"], ms["video"], ms["text"]], dim=1).cpu() + all_modality_scores.append(gate_tensor) + else: + all_modality_scores.append(ms.cpu()) + metrics = regression_metrics(y_true, y_pred) + if all_modality_scores: + avg_scores = torch.cat(all_modality_scores, dim=0).mean(dim=0).tolist() + if len(avg_scores) >= 3: + metrics["gate_audio"] = avg_scores[0] + metrics["gate_video"] = avg_scores[1] + metrics["gate_text"] = avg_scores[2] + return total_loss / max(1, len(self.val_loader)), metrics + + def _compute_batch_loss(self, batch: Dict[str, torch.Tensor]) -> torch.Tensor: + # Optional mixup augmentation — interpolate between random sample pairs + if self.model.training and self.mixup_alpha > 0.0 and len(batch["phq_total"]) > 1: + batch = self._apply_mixup(batch) + + output = self._forward(batch) + + # Use raw_weights (has gradients) so gate_balance loss can backprop + gate_weights = None + ms = output.get("modality_scores") + if ms is not None and isinstance(ms, dict): + raw = ms.get("raw_weights") + if raw is not None: + gate_weights = raw + else: + gate_weights = torch.stack( + [ms["audio"], ms["video"], ms["text"]], dim=1 + ).to(self.device) + + loss = self.loss_fn( + output["phq_total"], + batch["phq_total"].to(self.device), + output["phq_questions"], + batch["phq_questions"].to(self.device), + output["classification"], + batch["classification"].to(self.device), + output["entropy"], + confidence_pred=output.get("modality_confidence"), + gate_weights=gate_weights, + ) + return loss + + def _apply_mixup(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Mix pairs of samples in the batch (label-preserving interpolation).""" + import numpy as np + B = batch["phq_total"].shape[0] + lam = float(np.random.beta(self.mixup_alpha, self.mixup_alpha)) + idx = torch.randperm(B) + mixed = dict(batch) # shallow copy + for key in ("phq_total", "phq_questions", "classification"): + mixed[key] = lam * batch[key] + (1 - lam) * batch[key][idx] + # Feature tensors: mix along time dim with the same lambda + for key in ("audio", "video", "text"): + if batch.get(key) is not None: + mixed[key] = lam * batch[key] + (1 - lam) * batch[key][idx] + return mixed + + def _forward(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + audio = batch["audio"].to(self.device) if batch.get("audio") is not None else None + video = batch["video"].to(self.device) if batch.get("video") is not None else None + text = batch["text"].to(self.device) if batch.get("text") is not None else None + audio_mask = batch["audio_mask"].to(self.device) + video_mask = batch["video_mask"].to(self.device) + text_mask = batch["text_mask"].to(self.device) + return self.model(audio, video, text, audio_mask, video_mask, text_mask) diff --git a/src/utils/__init__.py b/src/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/utils/__pycache__/__init__.cpython-312.pyc b/src/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be34903791aa5d415e8958dae0022ec58e7a7ff7 Binary files /dev/null and b/src/utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/utils/__pycache__/config.cpython-312.pyc b/src/utils/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0b410db0152007e2c07330e679068cfef2fc9e80 Binary files /dev/null and b/src/utils/__pycache__/config.cpython-312.pyc differ diff --git a/src/utils/__pycache__/experiment.cpython-312.pyc b/src/utils/__pycache__/experiment.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb34ed07d2fe2a09c0e1caed243078c7d728968c Binary files /dev/null and b/src/utils/__pycache__/experiment.cpython-312.pyc differ diff --git a/src/utils/__pycache__/logging.cpython-312.pyc b/src/utils/__pycache__/logging.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..acdaa9afd6a2ea8f336004a9ac784f96553674e8 Binary files /dev/null and b/src/utils/__pycache__/logging.cpython-312.pyc differ diff --git a/src/utils/__pycache__/types.cpython-312.pyc b/src/utils/__pycache__/types.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e64292388fa22c70df77baf360047dbfc42d7554 Binary files /dev/null and b/src/utils/__pycache__/types.cpython-312.pyc differ diff --git a/src/utils/config.py b/src/utils/config.py new file mode 100644 index 0000000000000000000000000000000000000000..552b63444724cac16e9513883a7c72314384a24d --- /dev/null +++ b/src/utils/config.py @@ -0,0 +1,86 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Optional, Dict, Any + + +@dataclass(frozen=True) +class AudioConfig: + sample_rate: int = 16000 + frame_rate: int = 8 + chunk_duration: float = 2.0 + chunk_overlap: float = 0.5 + vad_mode: str = "aggressive" + feature_backends: tuple[str, ...] = ("mfcc", "egemaps", "hubert") + mfcc_n_mfcc: int = 13 + hubert_model_name: str = "facebook/hubert-base-ls960" + + +@dataclass(frozen=True) +class VideoConfig: + frame_rate: int = 8 + temporal_window: int = 32 + openface_csv_key: str = "openface" + embedding_backend: str = "resnet50" + encoder_type: str = "transformer" + + +@dataclass(frozen=True) +class TextConfig: + transformer_model: str = "sentence-transformers/all-mpnet-base-v2" + max_tokens: int = 256 + use_session_level: bool = True + use_chunk_level: bool = True + + +@dataclass(frozen=True) +class ModelConfig: + audio_dim: int = 512 + video_dim: int = 512 + text_dim: int = 384 + fusion_dim: int = 256 + num_questions: int = 8 + fusion_mode: str = "hybrid" + dropout: float = 0.2 + encoder_type: str = "transformer" + fusion_layers: int = 2 + + +@dataclass(frozen=True) +class TrainingConfig: + batch_size: int = 8 + num_workers: int = 2 + max_epochs: int = 50 + learning_rate: float = 3e-4 + weight_decay: float = 1e-4 + grad_clip_norm: float = 1.0 + early_stopping_patience: int = 8 + use_amp: bool = True + modality_dropout: float = 0.2 + temporal_dropout: float = 0.1 + seed: int = 42 + + +@dataclass(frozen=True) +class InferenceConfig: + device: str = "auto" + batch_size: int = 4 + sanitize_values: bool = True + allow_missing_modalities: bool = True + max_batch_items: int = 16 + + +@dataclass(frozen=True) +class PipelineConfig: + audio: AudioConfig = AudioConfig() + video: VideoConfig = VideoConfig() + text: TextConfig = TextConfig() + model: ModelConfig = ModelConfig() + training: TrainingConfig = TrainingConfig() + inference: InferenceConfig = InferenceConfig() + + +def load_config(config_path: Path) -> Dict[str, Any]: + import yaml + + with open(config_path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) or {} diff --git a/src/utils/experiment.py b/src/utils/experiment.py new file mode 100644 index 0000000000000000000000000000000000000000..03ae1d1c1254dc7b5e45e0d6bd63f39544fcf34f --- /dev/null +++ b/src/utils/experiment.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import json +import random +from dataclasses import dataclass, asdict +from pathlib import Path + +import numpy as np +import torch + + +@dataclass +class ExperimentConfig: + name: str + seed: int = 42 + run_directory: str = "runs" + + +class ExperimentManager: + def __init__(self, config: ExperimentConfig): + self.config = config + self.root = Path(config.run_directory) / config.name + self.root.mkdir(parents=True, exist_ok=True) + self.seed(config.seed) + + def seed(self, seed: int) -> None: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + + def save_config(self, filename: str = "config.json") -> Path: + path = self.root / filename + with open(path, "w", encoding="utf-8") as handle: + json.dump(asdict(self.config), handle, indent=2) + return path + + def checkpoint_path(self, name: str = "best_checkpoint.pt") -> Path: + return self.root / name diff --git a/src/utils/logging.py b/src/utils/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..625d08d11349d1f2e9c6f94509b4d830e41e17f0 --- /dev/null +++ b/src/utils/logging.py @@ -0,0 +1,36 @@ +import logging +from pathlib import Path +from datetime import datetime + + +def configure_logging(log_dir: Path, level: str = "INFO", experiment_name: str = "trimodal") -> Path: + runs_dir = log_dir / "runs" + runs_dir.mkdir(parents=True, exist_ok=True) + + # Prune old runs to keep only the last 10 + import shutil + existing_runs = sorted([d for d in runs_dir.iterdir() if d.is_dir()], key=lambda x: x.stat().st_mtime) + if len(existing_runs) >= 10: + for old_dir in existing_runs[:-9]: # keep 9 existing + 1 new = 10 + shutil.rmtree(old_dir, ignore_errors=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + run_dir = runs_dir / f"{experiment_name}_{timestamp}" + run_dir.mkdir(parents=True, exist_ok=True) + + formatter = logging.Formatter( + "%(asctime)s [%(levelname)s] %(name)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + root_logger = logging.getLogger() + root_logger.setLevel(level) + if not root_logger.handlers: + root_logger.addHandler(handler) + + file_handler = logging.FileHandler(run_dir / "training.log") + file_handler.setFormatter(formatter) + root_logger.addHandler(file_handler) + + return run_dir diff --git a/src/utils/types.py b/src/utils/types.py new file mode 100644 index 0000000000000000000000000000000000000000..70119a2e0afd78822decf033b49275669c365261 --- /dev/null +++ b/src/utils/types.py @@ -0,0 +1,16 @@ +from typing import TypedDict, Optional, Sequence, Any + + +class ModalityMask(TypedDict): + audio: Sequence[bool] + video: Sequence[bool] + text: Sequence[bool] + + +class InferenceResult(TypedDict): + phq_total: float + phq_questions: Sequence[float] + classification: float + confidence: float + modality_scores: dict[str, float] + metadata: Optional[dict[str, Any]]