Spaces:
Sleeping
Sleeping
Commit ·
8d7b4b1
1
Parent(s): 319bd68
Model Changes
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .dockerignore +28 -0
- Dockerfile +32 -0
- checkpoints/best_fusion.pt +3 -0
- checkpoints/best_model.pt +3 -0
- checkpoints/best_model_backup_0.14ccc.pt +3 -0
- checkpoints/best_stage1.pt +3 -0
- checkpoints/best_stage2_clean.pt +3 -0
- checkpoints/best_trimodal.pt +3 -0
- checkpoints/best_trimodal_backup.pt +3 -0
- checkpoints/best_trimodal_text.pt +3 -0
- checkpoints/cv/best_model_fold0.pt +3 -0
- checkpoints/cv/best_model_fold1.pt +3 -0
- checkpoints/cv/best_model_fold2.pt +3 -0
- checkpoints/cv/best_model_fold3.pt +3 -0
- checkpoints/cv/best_model_fold4.pt +3 -0
- checkpoints/multimodal_v2/feature_spec.json +32 -0
- checkpoints/multimodal_v2/preprocessors/audio_preprocessor.pkl +3 -0
- checkpoints/multimodal_v2/preprocessors/text_preprocessor.pkl +3 -0
- checkpoints/multimodal_v2/preprocessors/video_preprocessor.pkl +3 -0
- checkpoints/multimodal_v3/feature_spec.json +32 -0
- checkpoints/multimodal_v3/preprocessors/audio_preprocessor.pkl +3 -0
- checkpoints/multimodal_v3/preprocessors/text_preprocessor.pkl +3 -0
- checkpoints/multimodal_v3/preprocessors/video_preprocessor.pkl +3 -0
- checkpoints/multimodal_v4/feature_spec.json +32 -0
- checkpoints/multimodal_v4/preprocessors/audio_preprocessor.pkl +3 -0
- checkpoints/multimodal_v4/preprocessors/text_preprocessor.pkl +3 -0
- checkpoints/multimodal_v4/preprocessors/video_preprocessor.pkl +3 -0
- checkpoints/pca_cnn_transform.pkl +3 -0
- checkpoints/scalers/feature_scalers.pkl +3 -0
- checkpoints/scalers/pca_reducer.pkl +3 -0
- configs/feature_extraction.yaml +17 -0
- configs/inference.yaml +19 -0
- configs/training.yaml +55 -0
- requirements.txt +26 -0
- scripts/__pycache__/train.cpython-312.pyc +0 -0
- scripts/__pycache__/train_v2.cpython-312.pyc +0 -0
- scripts/auto_train.py +51 -0
- scripts/evaluate.py +68 -0
- scripts/extract_features.py +54 -0
- scripts/infer_live_v2.py +96 -0
- scripts/serve.py +37 -0
- scripts/smoke_infer_test.py +37 -0
- scripts/train.py +93 -0
- scripts/train_v2.py +378 -0
- src/__init__.py +0 -0
- src/__pycache__/__init__.cpython-312.pyc +0 -0
- src/api/__init__.py +0 -0
- src/api/__pycache__/__init__.cpython-312.pyc +0 -0
- src/api/__pycache__/app.cpython-312.pyc +0 -0
- src/api/__pycache__/routes.cpython-312.pyc +0 -0
.dockerignore
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.venv/
|
| 2 |
+
.git/
|
| 3 |
+
__pycache__/
|
| 4 |
+
*.pyc
|
| 5 |
+
*.pyo
|
| 6 |
+
logs*/
|
| 7 |
+
logs_v4_run.log
|
| 8 |
+
data/
|
| 9 |
+
notebooks/
|
| 10 |
+
docs/
|
| 11 |
+
runs/
|
| 12 |
+
*.zip
|
| 13 |
+
*.tar.gz
|
| 14 |
+
checkpoints/cv/
|
| 15 |
+
checkpoints/multimodal_v2/
|
| 16 |
+
checkpoints/multimodal_v3/
|
| 17 |
+
checkpoints/best_fusion.pt
|
| 18 |
+
checkpoints/best_model_backup_0.14ccc.pt
|
| 19 |
+
checkpoints/best_stage1.pt
|
| 20 |
+
checkpoints/best_stage2_clean.pt
|
| 21 |
+
checkpoints/best_trimodal.pt
|
| 22 |
+
checkpoints/best_trimodal_backup.pt
|
| 23 |
+
checkpoints/best_trimodal_text.pt
|
| 24 |
+
checkpoints/pca_cnn_transform.pkl
|
| 25 |
+
checkpoints/scalers/
|
| 26 |
+
windows/
|
| 27 |
+
.vscode/
|
| 28 |
+
tests/
|
Dockerfile
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.11-slim
|
| 2 |
+
|
| 3 |
+
WORKDIR /app
|
| 4 |
+
|
| 5 |
+
# System deps: libgl (mediapipe/opencv), ffmpeg (audio), libgomp (torch)
|
| 6 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 7 |
+
libgl1 \
|
| 8 |
+
libglib2.0-0 \
|
| 9 |
+
libsm6 \
|
| 10 |
+
libxext6 \
|
| 11 |
+
libxrender-dev \
|
| 12 |
+
libgomp1 \
|
| 13 |
+
ffmpeg \
|
| 14 |
+
git \
|
| 15 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 16 |
+
|
| 17 |
+
COPY requirements.txt .
|
| 18 |
+
|
| 19 |
+
# Install CPU-only torch — latest from wheel index (no version pin)
|
| 20 |
+
# Same approach as start-all.sh: installs latest compatible cpu set
|
| 21 |
+
RUN pip install --no-cache-dir \
|
| 22 |
+
torch torchaudio torchvision \
|
| 23 |
+
--index-url https://download.pytorch.org/whl/cpu
|
| 24 |
+
|
| 25 |
+
# Install remaining deps (torch already satisfied above, won't re-download)
|
| 26 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 27 |
+
|
| 28 |
+
COPY . .
|
| 29 |
+
|
| 30 |
+
EXPOSE 8001
|
| 31 |
+
|
| 32 |
+
CMD ["python", "scripts/serve.py", "--port", "8001"]
|
checkpoints/best_fusion.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4a3f27f622fcf180f1704d4612273a97813a85d8893b87e9a185d2915cdcbeda
|
| 3 |
+
size 43669
|
checkpoints/best_model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:147924ec16e02220048ff2c28ad71daafe3a93d35771ca9e4e4de1279c4e78c3
|
| 3 |
+
size 405941
|
checkpoints/best_model_backup_0.14ccc.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6f0ed9844a410329a275b9fc7bbdc3e89128ebeb292e568ae6695d329936177d
|
| 3 |
+
size 43125
|
checkpoints/best_stage1.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fc5946d5372483f4eaeadba08255ce16eaa7bf656b7ca15a0c5a8d12d1ea3551
|
| 3 |
+
size 349073
|
checkpoints/best_stage2_clean.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b966db1077d73ce7e88a8e212b3d16c3f5994d8d128007e69f944f6adf82c505
|
| 3 |
+
size 1870025
|
checkpoints/best_trimodal.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7c50f63feb8adcce165846a833ac72bf6601c2d96624b24b71d12ced38627b6f
|
| 3 |
+
size 1869605
|
checkpoints/best_trimodal_backup.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:39ab55d17b81e93f47cbf988a190fd26e8f6a83570f366321a4daf1835f4d046
|
| 3 |
+
size 1869255
|
checkpoints/best_trimodal_text.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:818cccc4fab056005b651ead9ef7bb5c430be39b31a291d3928a9002e9813405
|
| 3 |
+
size 1891597
|
checkpoints/cv/best_model_fold0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2783e75575e344833fbc9224a7191e2be957cdc6d6f3016378618b11ac6bc930
|
| 3 |
+
size 43339
|
checkpoints/cv/best_model_fold1.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a1b18a628524c2aa70398976ed90d36131ffbe7910675a5792f17540f244ea77
|
| 3 |
+
size 43339
|
checkpoints/cv/best_model_fold2.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9f616da338d0dadba40884826c4959d47afeae3a321ffc907445fa6afcb8d7e4
|
| 3 |
+
size 43339
|
checkpoints/cv/best_model_fold3.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:937ba7d3640334ef41b05f15bc4bc2581ba60f8fe3711ff9fe2f406154db05f7
|
| 3 |
+
size 43339
|
checkpoints/cv/best_model_fold4.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:44f2863172d498f05bd4b34fb7fa56b131cc74e9cf2c986d78537acc85fdbaf6
|
| 3 |
+
size 43339
|
checkpoints/multimodal_v2/feature_spec.json
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"audio": {
|
| 3 |
+
"name": "audio",
|
| 4 |
+
"expected_dim": 62,
|
| 5 |
+
"temporal": true,
|
| 6 |
+
"preprocessor_class": null,
|
| 7 |
+
"extractor_class": null,
|
| 8 |
+
"pca_enabled": false,
|
| 9 |
+
"pca_target_dim": null,
|
| 10 |
+
"normalizer_type": "standard"
|
| 11 |
+
},
|
| 12 |
+
"video": {
|
| 13 |
+
"name": "video",
|
| 14 |
+
"expected_dim": 49,
|
| 15 |
+
"temporal": true,
|
| 16 |
+
"preprocessor_class": null,
|
| 17 |
+
"extractor_class": null,
|
| 18 |
+
"pca_enabled": false,
|
| 19 |
+
"pca_target_dim": null,
|
| 20 |
+
"normalizer_type": "standard"
|
| 21 |
+
},
|
| 22 |
+
"text": {
|
| 23 |
+
"name": "text",
|
| 24 |
+
"expected_dim": 384,
|
| 25 |
+
"temporal": true,
|
| 26 |
+
"preprocessor_class": null,
|
| 27 |
+
"extractor_class": null,
|
| 28 |
+
"pca_enabled": false,
|
| 29 |
+
"pca_target_dim": null,
|
| 30 |
+
"normalizer_type": "standard"
|
| 31 |
+
}
|
| 32 |
+
}
|
checkpoints/multimodal_v2/preprocessors/audio_preprocessor.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bc6356119bc5f47417ee3f5fd57585a9c83e4258a83aa223c4ab34f73714b2af
|
| 3 |
+
size 11640
|
checkpoints/multimodal_v2/preprocessors/text_preprocessor.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ed802752135f85d0d847b33077e36492a27511ad35df1ca5cfcfa6b9a6af0226
|
| 3 |
+
size 264428
|
checkpoints/multimodal_v2/preprocessors/video_preprocessor.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:06b856001f26ddfe1b74400f476d6c2b80e000b67bd571f29e4b8857cd129df4
|
| 3 |
+
size 11016
|
checkpoints/multimodal_v3/feature_spec.json
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"audio": {
|
| 3 |
+
"name": "audio",
|
| 4 |
+
"expected_dim": 62,
|
| 5 |
+
"temporal": true,
|
| 6 |
+
"preprocessor_class": null,
|
| 7 |
+
"extractor_class": null,
|
| 8 |
+
"pca_enabled": false,
|
| 9 |
+
"pca_target_dim": null,
|
| 10 |
+
"normalizer_type": "standard"
|
| 11 |
+
},
|
| 12 |
+
"video": {
|
| 13 |
+
"name": "video",
|
| 14 |
+
"expected_dim": 49,
|
| 15 |
+
"temporal": true,
|
| 16 |
+
"preprocessor_class": null,
|
| 17 |
+
"extractor_class": null,
|
| 18 |
+
"pca_enabled": false,
|
| 19 |
+
"pca_target_dim": null,
|
| 20 |
+
"normalizer_type": "standard"
|
| 21 |
+
},
|
| 22 |
+
"text": {
|
| 23 |
+
"name": "text",
|
| 24 |
+
"expected_dim": 384,
|
| 25 |
+
"temporal": true,
|
| 26 |
+
"preprocessor_class": null,
|
| 27 |
+
"extractor_class": null,
|
| 28 |
+
"pca_enabled": false,
|
| 29 |
+
"pca_target_dim": null,
|
| 30 |
+
"normalizer_type": "standard"
|
| 31 |
+
}
|
| 32 |
+
}
|
checkpoints/multimodal_v3/preprocessors/audio_preprocessor.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:caff65d8a1812be62c39d70901e1801791631910cfda40e08443a04ae8efa30d
|
| 3 |
+
size 11733
|
checkpoints/multimodal_v3/preprocessors/text_preprocessor.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5ca025ec6356361869c37c00b5b5a4e5bf4b0307192801010889fc67ea495c8a
|
| 3 |
+
size 264521
|
checkpoints/multimodal_v3/preprocessors/video_preprocessor.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:963825c9464b72a6986599744b34fad3303529b2040773cfc2470e675d0650dd
|
| 3 |
+
size 11109
|
checkpoints/multimodal_v4/feature_spec.json
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"audio": {
|
| 3 |
+
"name": "audio",
|
| 4 |
+
"expected_dim": 62,
|
| 5 |
+
"temporal": true,
|
| 6 |
+
"preprocessor_class": null,
|
| 7 |
+
"extractor_class": null,
|
| 8 |
+
"pca_enabled": false,
|
| 9 |
+
"pca_target_dim": null,
|
| 10 |
+
"normalizer_type": "standard"
|
| 11 |
+
},
|
| 12 |
+
"video": {
|
| 13 |
+
"name": "video",
|
| 14 |
+
"expected_dim": 49,
|
| 15 |
+
"temporal": true,
|
| 16 |
+
"preprocessor_class": null,
|
| 17 |
+
"extractor_class": null,
|
| 18 |
+
"pca_enabled": false,
|
| 19 |
+
"pca_target_dim": null,
|
| 20 |
+
"normalizer_type": "standard"
|
| 21 |
+
},
|
| 22 |
+
"text": {
|
| 23 |
+
"name": "text",
|
| 24 |
+
"expected_dim": 384,
|
| 25 |
+
"temporal": true,
|
| 26 |
+
"preprocessor_class": null,
|
| 27 |
+
"extractor_class": null,
|
| 28 |
+
"pca_enabled": false,
|
| 29 |
+
"pca_target_dim": null,
|
| 30 |
+
"normalizer_type": "standard"
|
| 31 |
+
}
|
| 32 |
+
}
|
checkpoints/multimodal_v4/preprocessors/audio_preprocessor.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:caff65d8a1812be62c39d70901e1801791631910cfda40e08443a04ae8efa30d
|
| 3 |
+
size 11733
|
checkpoints/multimodal_v4/preprocessors/text_preprocessor.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5ca025ec6356361869c37c00b5b5a4e5bf4b0307192801010889fc67ea495c8a
|
| 3 |
+
size 264521
|
checkpoints/multimodal_v4/preprocessors/video_preprocessor.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:963825c9464b72a6986599744b34fad3303529b2040773cfc2470e675d0650dd
|
| 3 |
+
size 11109
|
checkpoints/pca_cnn_transform.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:654fe52593b0a178def46f0cb3479dd4558e157233b9a7f3ed031df0a98b131d
|
| 3 |
+
size 1059084
|
checkpoints/scalers/feature_scalers.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:878445280c1c0c5d1799346f07d08d0e915d27ed4b1d2ed1663c99ca440250d1
|
| 3 |
+
size 1942
|
checkpoints/scalers/pca_reducer.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:17451fb4ae35aeb65c907cc3dd78ca2663f8f57473675e55d1d253e1e2300227
|
| 3 |
+
size 120065
|
configs/feature_extraction.yaml
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
audio:
|
| 2 |
+
sample_rate: 16000
|
| 3 |
+
frame_rate: 8
|
| 4 |
+
chunk_duration: 2.0
|
| 5 |
+
chunk_overlap: 0.5
|
| 6 |
+
mfcc_n_mfcc: 13
|
| 7 |
+
hubert_model_name: facebook/hubert-base-ls960
|
| 8 |
+
feature_backends:
|
| 9 |
+
- mfcc
|
| 10 |
+
- hubert
|
| 11 |
+
video:
|
| 12 |
+
frame_rate: 8
|
| 13 |
+
temporal_window: 32
|
| 14 |
+
openface_csv_key: openface
|
| 15 |
+
embedding_backend: resnet50
|
| 16 |
+
text:
|
| 17 |
+
transformer_model: sentence-transformers/all-mpnet-base-v2
|
configs/inference.yaml
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
artifacts:
|
| 2 |
+
checkpoint_path: checkpoints/best_model.pt
|
| 3 |
+
model:
|
| 4 |
+
audio_dim: 33
|
| 5 |
+
video_dim: 40
|
| 6 |
+
text_dim: 163
|
| 7 |
+
fusion_dim: 32
|
| 8 |
+
num_questions: 8
|
| 9 |
+
fusion_mode: hybrid
|
| 10 |
+
encoder_type: conv_gru
|
| 11 |
+
num_layers: 1
|
| 12 |
+
dropout: 0.1
|
| 13 |
+
inference:
|
| 14 |
+
device: auto
|
| 15 |
+
batch_size: 4
|
| 16 |
+
api:
|
| 17 |
+
port: 8000
|
| 18 |
+
logging:
|
| 19 |
+
log_dir: logs
|
configs/training.yaml
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset:
|
| 2 |
+
feature_dir: data/npz
|
| 3 |
+
id_column: participant_id
|
| 4 |
+
label_column: phq_score
|
| 5 |
+
train_split: data/splits/train_split.csv
|
| 6 |
+
val_split: data/splits/dev_split.csv
|
| 7 |
+
features:
|
| 8 |
+
audio_dim: 62
|
| 9 |
+
text_dim: 384
|
| 10 |
+
video_dim: 49
|
| 11 |
+
logging:
|
| 12 |
+
experiment_name: multimodal_v4
|
| 13 |
+
log_dir: logs_v4
|
| 14 |
+
loss:
|
| 15 |
+
# Keep exact weights from multimodal_v3 that achieved CCC=0.5058
|
| 16 |
+
ccc_weight: 30.0
|
| 17 |
+
regression_weight: 0.3
|
| 18 |
+
question_weight: 0.25
|
| 19 |
+
classification_weight: 0.05
|
| 20 |
+
confidence_weight: 0.01
|
| 21 |
+
entropy_weight: 0.2
|
| 22 |
+
# Zero gate_balance → text can freely exceed 1/3 without competing loss
|
| 23 |
+
gate_balance_weight: 0.0
|
| 24 |
+
# Text gate preference — penalise when text gate < 0.40
|
| 25 |
+
text_gate_weight: 0.25
|
| 26 |
+
text_gate_target: 0.40
|
| 27 |
+
model:
|
| 28 |
+
# Exact dims from multimodal_v3 (proven CCC=0.5058, 91K params)
|
| 29 |
+
audio_dim: 33
|
| 30 |
+
video_dim: 40
|
| 31 |
+
text_dim: 163
|
| 32 |
+
dropout: 0.45
|
| 33 |
+
encoder_type: conv_gru
|
| 34 |
+
fusion_dim: 32
|
| 35 |
+
fusion_mode: hybrid
|
| 36 |
+
modality_dropout: 0.20
|
| 37 |
+
num_heads: 4
|
| 38 |
+
num_layers: 1
|
| 39 |
+
num_questions: 8
|
| 40 |
+
regularization:
|
| 41 |
+
feature_noise_std: 0.06
|
| 42 |
+
temporal_dropout_rate: 0.20
|
| 43 |
+
training:
|
| 44 |
+
batch_size: 16
|
| 45 |
+
early_stopping_min_delta: 0.001
|
| 46 |
+
early_stopping_patience: 200
|
| 47 |
+
grad_clip_norm: 1.0
|
| 48 |
+
learning_rate: 0.001
|
| 49 |
+
lr_factor: 0.5
|
| 50 |
+
lr_patience: 30
|
| 51 |
+
max_epochs: 1000
|
| 52 |
+
num_workers: 0
|
| 53 |
+
use_amp: true
|
| 54 |
+
weight_decay: 8e-3
|
| 55 |
+
mixup_alpha: 0.3
|
requirements.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch>=2.1.2
|
| 2 |
+
torchaudio>=2.1.2
|
| 3 |
+
torchvision>=0.16.2
|
| 4 |
+
numpy>=1.24.4
|
| 5 |
+
scikit-learn>=1.3.2
|
| 6 |
+
pandas>=2.0.3
|
| 7 |
+
librosa>=0.10.1
|
| 8 |
+
soundfile>=0.12.1
|
| 9 |
+
pydub>=0.25.1
|
| 10 |
+
opensmile>=2.4.2
|
| 11 |
+
transformers>=4.36.2,<5.6
|
| 12 |
+
sentence-transformers>=2.2.2,<5.5
|
| 13 |
+
fastapi>=0.104.1
|
| 14 |
+
uvicorn>=0.24.0
|
| 15 |
+
slowapi>=0.1.9
|
| 16 |
+
python-dotenv>=1.0.0
|
| 17 |
+
pyyaml>=6.0.1
|
| 18 |
+
sqlalchemy>=2.0.23
|
| 19 |
+
tqdm>=4.66.1
|
| 20 |
+
joblib>=1.3.2
|
| 21 |
+
pydantic>=2.5.3
|
| 22 |
+
requests>=2.31.0
|
| 23 |
+
python-multipart>=0.0.6
|
| 24 |
+
scipy>=1.11.4
|
| 25 |
+
h5py>=3.10.0
|
| 26 |
+
matplotlib>=3.8.2
|
scripts/__pycache__/train.cpython-312.pyc
ADDED
|
Binary file (4.97 kB). View file
|
|
|
scripts/__pycache__/train_v2.cpython-312.pyc
ADDED
|
Binary file (17 kB). View file
|
|
|
scripts/auto_train.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import subprocess
|
| 3 |
+
import yaml
|
| 4 |
+
|
| 5 |
+
config_path = "configs/training.yaml"
|
| 6 |
+
|
| 7 |
+
def run_training():
|
| 8 |
+
print("Running training...")
|
| 9 |
+
result = subprocess.run(["python3", "scripts/train_v2.py", "--config", config_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
| 10 |
+
|
| 11 |
+
best_ccc = -1.0
|
| 12 |
+
for line in result.stdout.split('\n'):
|
| 13 |
+
if "--> New best model saved with CCC:" in line:
|
| 14 |
+
try:
|
| 15 |
+
ccc = float(line.split(":")[-1].strip())
|
| 16 |
+
if ccc > best_ccc:
|
| 17 |
+
best_ccc = ccc
|
| 18 |
+
except:
|
| 19 |
+
pass
|
| 20 |
+
print(f"Training finished. Best CCC: {best_ccc}")
|
| 21 |
+
if best_ccc == -1.0:
|
| 22 |
+
print("Training failed or did not report CCC. Output:")
|
| 23 |
+
print(result.stdout)
|
| 24 |
+
return best_ccc
|
| 25 |
+
|
| 26 |
+
def update_config(lr, dropout, ccc_weight):
|
| 27 |
+
with open(config_path, "r") as f:
|
| 28 |
+
config = yaml.safe_load(f)
|
| 29 |
+
|
| 30 |
+
config["model"]["dropout"] = dropout
|
| 31 |
+
config["training"]["learning_rate"] = lr
|
| 32 |
+
config["loss"]["ccc_weight"] = ccc_weight
|
| 33 |
+
|
| 34 |
+
with open(config_path, "w") as f:
|
| 35 |
+
yaml.safe_dump(config, f)
|
| 36 |
+
|
| 37 |
+
lrs = [5e-4, 1e-4, 8e-4]
|
| 38 |
+
dropouts = [0.1, 0.2, 0.3]
|
| 39 |
+
ccc_weights = [1.5, 2.0, 3.0]
|
| 40 |
+
|
| 41 |
+
for lr in lrs:
|
| 42 |
+
for drop in dropouts:
|
| 43 |
+
for cw in ccc_weights:
|
| 44 |
+
print(f"Testing LR: {lr}, Dropout: {drop}, CCC Weight: {cw}")
|
| 45 |
+
update_config(lr, drop, cw)
|
| 46 |
+
ccc = run_training()
|
| 47 |
+
if ccc >= 0.55:
|
| 48 |
+
print(f"Success! Reached {ccc} CCC with LR={lr}, Drop={drop}, CCC Weight={cw}")
|
| 49 |
+
exit(0)
|
| 50 |
+
|
| 51 |
+
print("Failed to reach 0.55 CCC after all trials.")
|
scripts/evaluate.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import logging
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
ROOT_DIR = Path(__file__).resolve().parents[1]
|
| 7 |
+
if str(ROOT_DIR) not in sys.path:
|
| 8 |
+
sys.path.insert(0, str(ROOT_DIR))
|
| 9 |
+
|
| 10 |
+
import yaml
|
| 11 |
+
import torch
|
| 12 |
+
from torch.utils.data import DataLoader
|
| 13 |
+
|
| 14 |
+
from src.dataset.builder import DatasetBuilder
|
| 15 |
+
from src.dataset.collate import multimodal_collate_fn
|
| 16 |
+
from src.models.multimodal_model import MultimodalDepressionModel
|
| 17 |
+
from src.training.trainer import Trainer
|
| 18 |
+
from src.utils.logging import configure_logging
|
| 19 |
+
from src.evaluation.evaluator import evaluate_predictions
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def load_config(path: Path) -> dict:
|
| 23 |
+
with open(path, "r", encoding="utf-8") as handle:
|
| 24 |
+
return yaml.safe_load(handle)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def main() -> None:
|
| 28 |
+
parser = argparse.ArgumentParser(description="Evaluate ModelV2 checkpoint")
|
| 29 |
+
parser.add_argument("--config", type=Path, default=Path("configs/inference.yaml"))
|
| 30 |
+
parser.add_argument("--checkpoint", type=Path, required=True)
|
| 31 |
+
args = parser.parse_args()
|
| 32 |
+
|
| 33 |
+
config = load_config(args.config)
|
| 34 |
+
configure_logging(Path(config.get("logging", {}).get("log_dir", "logs")))
|
| 35 |
+
logger = logging.getLogger(__name__)
|
| 36 |
+
|
| 37 |
+
val_dataset = DatasetBuilder.build_from_csv(
|
| 38 |
+
split_csv_path=Path(config["dataset"]["val_split"]),
|
| 39 |
+
feature_dir=Path(config["dataset"]["feature_dir"]),
|
| 40 |
+
id_column=config["dataset"].get("id_column", "participant_id"),
|
| 41 |
+
label_column=config["dataset"].get("label_column", "phq_total"),
|
| 42 |
+
augment=False,
|
| 43 |
+
)
|
| 44 |
+
val_loader = DataLoader(val_dataset, batch_size=config.get("inference", {}).get("batch_size", 4), shuffle=False, collate_fn=multimodal_collate_fn)
|
| 45 |
+
|
| 46 |
+
model = MultimodalDepressionModel(**config["model"])
|
| 47 |
+
model.load_state_dict(torch.load(args.checkpoint, map_location="cpu"))
|
| 48 |
+
model.eval()
|
| 49 |
+
|
| 50 |
+
y_true, y_pred, q_true, q_pred, cls_true, cls_pred = [], [], [], [], [], []
|
| 51 |
+
with torch.no_grad():
|
| 52 |
+
for batch in val_loader:
|
| 53 |
+
output = model(
|
| 54 |
+
batch["audio"], batch["video"], batch["text"], batch["audio_mask"], batch["video_mask"], batch["text_mask"],
|
| 55 |
+
)
|
| 56 |
+
y_true.extend(batch["phq_total"].tolist())
|
| 57 |
+
y_pred.extend(output["phq_total"].tolist())
|
| 58 |
+
q_true.extend(batch["phq_questions"].tolist())
|
| 59 |
+
q_pred.extend(output["phq_questions"].tolist())
|
| 60 |
+
cls_true.extend(batch["classification"].tolist())
|
| 61 |
+
cls_pred.extend(torch.sigmoid(output["classification"]).tolist())
|
| 62 |
+
|
| 63 |
+
results = evaluate_predictions(y_true, y_pred, q_true, q_pred, cls_true, cls_pred)
|
| 64 |
+
logger.info(f"Evaluation results: {results}")
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
if __name__ == "__main__":
|
| 68 |
+
main()
|
scripts/extract_features.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
from typing import Dict, Any
|
| 4 |
+
|
| 5 |
+
import yaml
|
| 6 |
+
|
| 7 |
+
from src.features.audio_features import AudioFeatureExtractor
|
| 8 |
+
from src.features.video_features import VideoFeatureExtractor
|
| 9 |
+
from src.features.text_features import TextFeatureExtractor
|
| 10 |
+
from src.features.feature_store import save_features
|
| 11 |
+
from src.preprocessing.audio_preprocessor import AudioPreprocessor
|
| 12 |
+
from src.preprocessing.text_preprocessor import TextPreprocessor
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def load_config(path: Path) -> Dict[str, Any]:
|
| 16 |
+
with open(path, "r", encoding="utf-8") as handle:
|
| 17 |
+
return yaml.safe_load(handle)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def main() -> None:
|
| 21 |
+
parser = argparse.ArgumentParser(description="Extract ModelV2 multimodal features")
|
| 22 |
+
parser.add_argument("--config", type=Path, default=Path("configs/feature_extraction.yaml"))
|
| 23 |
+
parser.add_argument("--input-dir", type=Path, required=True)
|
| 24 |
+
parser.add_argument("--output-dir", type=Path, required=True)
|
| 25 |
+
args = parser.parse_args()
|
| 26 |
+
|
| 27 |
+
config = load_config(args.config)
|
| 28 |
+
audio_preprocessor = AudioPreprocessor(**config["audio"])
|
| 29 |
+
audio_extractor = AudioFeatureExtractor(config["audio"])
|
| 30 |
+
video_extractor = VideoFeatureExtractor(config["video"])
|
| 31 |
+
text_preprocessor = TextPreprocessor()
|
| 32 |
+
text_extractor = TextFeatureExtractor(config["text"]["transformer_model"])
|
| 33 |
+
|
| 34 |
+
for subject_dir in args.input_dir.iterdir():
|
| 35 |
+
if not subject_dir.is_dir():
|
| 36 |
+
continue
|
| 37 |
+
participant_id = subject_dir.name
|
| 38 |
+
audio_path = subject_dir / "audio.wav"
|
| 39 |
+
transcript_path = subject_dir / "transcript.txt"
|
| 40 |
+
openface_path = subject_dir / "openface.csv"
|
| 41 |
+
if not audio_path.exists() or not transcript_path.exists() or not openface_path.exists():
|
| 42 |
+
continue
|
| 43 |
+
waveform = audio_preprocessor.load_audio(audio_path)
|
| 44 |
+
audio_chunks = audio_preprocessor.preprocess(waveform)
|
| 45 |
+
audio_features = {"audio": audio_extractor.extract_modern(waveform)}
|
| 46 |
+
video_behavior = video_extractor.extract_openface(openface_path)["behavior"]
|
| 47 |
+
text = transcript_path.read_text(encoding="utf-8")
|
| 48 |
+
chunks = text_preprocessor.preprocess(text)
|
| 49 |
+
text_features = {"text": text_extractor.encode(chunks)}
|
| 50 |
+
save_features(args.output_dir / f"{participant_id}.npz", {
|
| 51 |
+
**audio_features,
|
| 52 |
+
"video": video_behavior,
|
| 53 |
+
**text_features,
|
| 54 |
+
})
|
scripts/infer_live_v2.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Example live inference script for multimodal depression detection.
|
| 3 |
+
|
| 4 |
+
Demonstrates how to use the LiveInferencePipeline for real-time predictions
|
| 5 |
+
from webcam/video/audio/transcript inputs.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import argparse
|
| 9 |
+
import sys
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
ROOT_DIR = Path(__file__).resolve().parents[2]
|
| 13 |
+
if str(ROOT_DIR) not in sys.path:
|
| 14 |
+
sys.path.insert(0, str(ROOT_DIR))
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
from src.inference.live_pipeline import LiveInferencePipeline
|
| 18 |
+
from src.features.specs import FeatureSpecification
|
| 19 |
+
from src.inference.validation import FeatureCheckpoint
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def main() -> None:
|
| 23 |
+
"""Example usage of live inference pipeline."""
|
| 24 |
+
parser = argparse.ArgumentParser(description="Live inference example")
|
| 25 |
+
parser.add_argument("--checkpoint", type=Path, required=True, help="Model checkpoint path")
|
| 26 |
+
parser.add_argument("--experiment", type=str, required=True, help="Experiment name for loading preprocessors")
|
| 27 |
+
parser.add_argument("--audio", type=Path, help="Path to audio file")
|
| 28 |
+
parser.add_argument("--openface-csv", type=Path, help="Path to OpenFace CSV (video features)")
|
| 29 |
+
parser.add_argument("--transcript", type=str, help="Transcript text")
|
| 30 |
+
parser.add_argument("--device", type=str, default="auto", help="Device (auto/cuda/cpu)")
|
| 31 |
+
parser.add_argument("--verbose", action="store_true", help="Verbose output")
|
| 32 |
+
args = parser.parse_args()
|
| 33 |
+
|
| 34 |
+
# Load feature spec and preprocessors
|
| 35 |
+
checkpoint_manager = FeatureCheckpoint(Path("checkpoints"))
|
| 36 |
+
|
| 37 |
+
try:
|
| 38 |
+
feature_spec = checkpoint_manager.load_feature_spec(args.experiment)
|
| 39 |
+
preprocessors = checkpoint_manager.load_preprocessors(args.experiment)
|
| 40 |
+
except FileNotFoundError as e:
|
| 41 |
+
print(f"Error: {e}")
|
| 42 |
+
print(f"Make sure to run training script with --experiment-name {args.experiment}")
|
| 43 |
+
sys.exit(1)
|
| 44 |
+
|
| 45 |
+
# Model config (same as training)
|
| 46 |
+
model_config = {
|
| 47 |
+
"audio_dim": feature_spec.audio_dim,
|
| 48 |
+
"video_dim": feature_spec.video_dim,
|
| 49 |
+
"text_dim": feature_spec.text_dim,
|
| 50 |
+
"fusion_dim": 256,
|
| 51 |
+
"num_questions": 8,
|
| 52 |
+
"encoder_type": "transformer",
|
| 53 |
+
"num_heads": 4,
|
| 54 |
+
"num_layers": 2,
|
| 55 |
+
"dropout": 0.2,
|
| 56 |
+
"fusion_mode": "hybrid",
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
# Initialize pipeline
|
| 60 |
+
pipeline = LiveInferencePipeline(
|
| 61 |
+
checkpoint_path=args.checkpoint,
|
| 62 |
+
model_config=model_config,
|
| 63 |
+
feature_spec=feature_spec,
|
| 64 |
+
preprocessors=preprocessors,
|
| 65 |
+
device=args.device,
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
if args.verbose:
|
| 69 |
+
print(f"Model loaded from: {args.checkpoint}")
|
| 70 |
+
print(f"Feature spec: audio={feature_spec.audio_dim}D, video={feature_spec.video_dim}D, text={feature_spec.text_dim}D")
|
| 71 |
+
|
| 72 |
+
# Run inference
|
| 73 |
+
try:
|
| 74 |
+
result = pipeline.predict_single(
|
| 75 |
+
audio_path=args.audio,
|
| 76 |
+
video_openface_csv=args.openface_csv,
|
| 77 |
+
transcript=args.transcript,
|
| 78 |
+
verbose=args.verbose,
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
# Print results
|
| 82 |
+
print(pipeline.predict_and_explain(
|
| 83 |
+
audio_path=args.audio,
|
| 84 |
+
video_openface_csv=args.openface_csv,
|
| 85 |
+
transcript=args.transcript,
|
| 86 |
+
))
|
| 87 |
+
|
| 88 |
+
except Exception as e:
|
| 89 |
+
print(f"Inference error: {e}")
|
| 90 |
+
import traceback
|
| 91 |
+
traceback.print_exc()
|
| 92 |
+
sys.exit(1)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
if __name__ == "__main__":
|
| 96 |
+
main()
|
scripts/serve.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import uvicorn
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
ROOT_DIR = Path(__file__).resolve().parents[1]
|
| 7 |
+
if str(ROOT_DIR) not in sys.path:
|
| 8 |
+
sys.path.insert(0, str(ROOT_DIR))
|
| 9 |
+
|
| 10 |
+
from src.api.app import create_app
|
| 11 |
+
from src.utils.config import load_config
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def main() -> None:
|
| 15 |
+
parser = argparse.ArgumentParser(description="Serve ModelV2 inference API")
|
| 16 |
+
parser.add_argument("--config", type=Path, default=Path("configs/inference.yaml"))
|
| 17 |
+
parser.add_argument("--host", type=str, default="0.0.0.0")
|
| 18 |
+
parser.add_argument("--port", type=int, default=None)
|
| 19 |
+
parser.add_argument("--log-level", type=str, default="info")
|
| 20 |
+
parser.add_argument("--reload", action="store_true")
|
| 21 |
+
args = parser.parse_args()
|
| 22 |
+
config = load_config(args.config)
|
| 23 |
+
if args.port is not None:
|
| 24 |
+
config["api"]["port"] = args.port
|
| 25 |
+
checkpoint_path = config["artifacts"]["checkpoint_path"]
|
| 26 |
+
model_config = config["model"]
|
| 27 |
+
device = config.get("inference", {}).get("device", "auto")
|
| 28 |
+
app = create_app(str(checkpoint_path), model_config, device=device)
|
| 29 |
+
|
| 30 |
+
# We must pass the import string 'src.api.app:create_app' if we want reload to work,
|
| 31 |
+
# but since app is already created, reload might not work properly without restructuring.
|
| 32 |
+
# We'll just pass the app directly.
|
| 33 |
+
uvicorn.run(app, host=args.host, port=config.get("api", {}).get("port", 8000), log_level=args.log_level.lower())
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
if __name__ == "__main__":
|
| 37 |
+
main()
|
scripts/smoke_infer_test.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Smoke test: load inference config and instantiate ModelV2Inferencer using CPU."""
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import traceback
|
| 6 |
+
|
| 7 |
+
from src.utils.config import load_config
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def main():
|
| 11 |
+
try:
|
| 12 |
+
repo_root = Path(__file__).resolve().parents[1]
|
| 13 |
+
cfg_path = repo_root / "configs" / "inference.yaml"
|
| 14 |
+
cfg = load_config(cfg_path)
|
| 15 |
+
# Force CPU to avoid CUDA/device autodetection issues
|
| 16 |
+
cfg.setdefault("inference", {})["device"] = "cpu"
|
| 17 |
+
|
| 18 |
+
checkpoint = Path(cfg["artifacts"]["checkpoint_path"]) if "artifacts" in cfg else Path("checkpoints/best_model.pt")
|
| 19 |
+
if not checkpoint.is_absolute():
|
| 20 |
+
checkpoint = (repo_root / checkpoint).resolve()
|
| 21 |
+
|
| 22 |
+
print("Config loaded. Checkpoint:", checkpoint)
|
| 23 |
+
print("Exists:", checkpoint.exists())
|
| 24 |
+
|
| 25 |
+
from src.inference.inferencer import ModelV2Inferencer
|
| 26 |
+
|
| 27 |
+
model_cfg = cfg.get("model", {})
|
| 28 |
+
infer = ModelV2Inferencer(checkpoint, model_cfg, device="cpu")
|
| 29 |
+
print("Model instantiated on device:", infer.device)
|
| 30 |
+
print("Smoke test OK")
|
| 31 |
+
except Exception:
|
| 32 |
+
traceback.print_exc()
|
| 33 |
+
sys.exit(2)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
if __name__ == "__main__":
|
| 37 |
+
main()
|
scripts/train.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import logging
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
ROOT_DIR = Path(__file__).resolve().parents[1]
|
| 7 |
+
if str(ROOT_DIR) not in sys.path:
|
| 8 |
+
sys.path.insert(0, str(ROOT_DIR))
|
| 9 |
+
|
| 10 |
+
import yaml
|
| 11 |
+
import torch
|
| 12 |
+
from torch.utils.data import DataLoader
|
| 13 |
+
|
| 14 |
+
from src.dataset.builder import DatasetBuilder
|
| 15 |
+
from src.dataset.collate import multimodal_collate_fn
|
| 16 |
+
from src.models.multimodal_model import MultimodalDepressionModel
|
| 17 |
+
from src.training.trainer import Trainer
|
| 18 |
+
from src.utils.logging import configure_logging
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def load_config(path: Path) -> dict:
|
| 22 |
+
with open(path, "r", encoding="utf-8") as handle:
|
| 23 |
+
return yaml.safe_load(handle)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def main() -> None:
|
| 27 |
+
parser = argparse.ArgumentParser(description="Train ModelV2 multimodal depression model")
|
| 28 |
+
parser.add_argument("--config", type=Path, default=Path("configs/training.yaml"))
|
| 29 |
+
args = parser.parse_args()
|
| 30 |
+
|
| 31 |
+
config = load_config(args.config)
|
| 32 |
+
run_dir = configure_logging(Path(config.get("logging", {}).get("log_dir", "logs")), experiment_name=config.get("logging", {}).get("experiment_name", "trimodal"))
|
| 33 |
+
logger = logging.getLogger(__name__)
|
| 34 |
+
|
| 35 |
+
train_dataset = DatasetBuilder.build_from_csv(
|
| 36 |
+
split_csv_path=Path(config["dataset"]["train_split"]),
|
| 37 |
+
feature_dir=Path(config["dataset"]["feature_dir"]),
|
| 38 |
+
id_column=config["dataset"].get("id_column", "participant_id"),
|
| 39 |
+
label_column=config["dataset"].get("label_column", "phq_total"),
|
| 40 |
+
augment=True,
|
| 41 |
+
)
|
| 42 |
+
val_dataset = DatasetBuilder.build_from_csv(
|
| 43 |
+
split_csv_path=Path(config["dataset"]["val_split"]),
|
| 44 |
+
feature_dir=Path(config["dataset"]["feature_dir"]),
|
| 45 |
+
id_column=config["dataset"].get("id_column", "participant_id"),
|
| 46 |
+
label_column=config["dataset"].get("label_column", "phq_total"),
|
| 47 |
+
augment=False,
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
train_loader = DataLoader(
|
| 51 |
+
train_dataset,
|
| 52 |
+
batch_size=int(config["training"].get("batch_size", 8)),
|
| 53 |
+
shuffle=True,
|
| 54 |
+
num_workers=int(config["training"].get("num_workers", 2)),
|
| 55 |
+
collate_fn=multimodal_collate_fn,
|
| 56 |
+
)
|
| 57 |
+
val_loader = DataLoader(
|
| 58 |
+
val_dataset,
|
| 59 |
+
batch_size=int(config["training"].get("batch_size", 8)),
|
| 60 |
+
shuffle=False,
|
| 61 |
+
num_workers=int(config["training"].get("num_workers", 2)),
|
| 62 |
+
collate_fn=multimodal_collate_fn,
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
model_config = config.get("model", {})
|
| 66 |
+
model = MultimodalDepressionModel(**model_config)
|
| 67 |
+
optimizer = torch.optim.AdamW(
|
| 68 |
+
model.parameters(),
|
| 69 |
+
lr=float(config["training"].get("learning_rate", 3e-4)),
|
| 70 |
+
weight_decay=float(config["training"].get("weight_decay", 1e-4)),
|
| 71 |
+
)
|
| 72 |
+
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
|
| 73 |
+
optimizer,
|
| 74 |
+
mode="min",
|
| 75 |
+
factor=float(config["training"].get("lr_factor", 0.5)),
|
| 76 |
+
patience=int(config["training"].get("lr_patience", 3)),
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
trainer = Trainer(
|
| 80 |
+
model=model,
|
| 81 |
+
train_loader=train_loader,
|
| 82 |
+
val_loader=val_loader,
|
| 83 |
+
optimizer=optimizer,
|
| 84 |
+
scheduler=scheduler,
|
| 85 |
+
config=config["training"],
|
| 86 |
+
)
|
| 87 |
+
logger.info("Starting ModelV2 training")
|
| 88 |
+
history = trainer.train()
|
| 89 |
+
logger.info(f"Training complete: {history}")
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
if __name__ == "__main__":
|
| 93 |
+
main()
|
scripts/train_v2.py
ADDED
|
@@ -0,0 +1,378 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Improved training script for multimodal depression detection.
|
| 3 |
+
|
| 4 |
+
Features:
|
| 5 |
+
- Consistent feature pipeline with validation
|
| 6 |
+
- Advanced regularization (EMA, checkpoint averaging, curriculum learning)
|
| 7 |
+
- Comprehensive logging and checkpointing
|
| 8 |
+
- Small dataset optimization
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import logging
|
| 13 |
+
import sys
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
import numpy as np
|
| 16 |
+
|
| 17 |
+
def convert_numpy(obj):
|
| 18 |
+
if isinstance(obj, np.integer):
|
| 19 |
+
return int(obj)
|
| 20 |
+
if isinstance(obj, np.floating):
|
| 21 |
+
return float(obj)
|
| 22 |
+
if isinstance(obj, np.ndarray):
|
| 23 |
+
return obj.tolist()
|
| 24 |
+
if isinstance(obj, dict):
|
| 25 |
+
return {k: convert_numpy(v) for k, v in obj.items()}
|
| 26 |
+
if isinstance(obj, list):
|
| 27 |
+
return [convert_numpy(i) for i in obj]
|
| 28 |
+
return obj
|
| 29 |
+
|
| 30 |
+
ROOT_DIR = Path(__file__).resolve().parents[1]
|
| 31 |
+
if str(ROOT_DIR) not in sys.path:
|
| 32 |
+
sys.path.insert(0, str(ROOT_DIR))
|
| 33 |
+
|
| 34 |
+
import yaml
|
| 35 |
+
import torch
|
| 36 |
+
from torch.utils.data import DataLoader
|
| 37 |
+
|
| 38 |
+
from src.dataset.builder import DatasetBuilder
|
| 39 |
+
from src.dataset.collate import multimodal_collate_fn
|
| 40 |
+
from src.dataset.multimodal_dataset import MultimodalSample
|
| 41 |
+
from src.models.multimodal_model import MultimodalDepressionModel
|
| 42 |
+
from src.training.trainer import Trainer
|
| 43 |
+
from src.training.optimization import EMA, WarmupScheduler, CurriculumLearning
|
| 44 |
+
from src.features.specs import FeatureSpecification
|
| 45 |
+
from src.features.preprocessing import FeaturePreprocessor
|
| 46 |
+
from src.inference.validation import TrainInferenceConsistencyChecker, FeatureCheckpoint
|
| 47 |
+
from src.utils.logging import configure_logging
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def load_config(path: Path) -> dict:
|
| 51 |
+
"""Load YAML configuration."""
|
| 52 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 53 |
+
return yaml.safe_load(f)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def setup_feature_specs(config: dict) -> FeatureSpecification:
|
| 57 |
+
"""Set up feature specifications from config."""
|
| 58 |
+
# Use defaults or override from config
|
| 59 |
+
feature_config = config.get("features", {})
|
| 60 |
+
|
| 61 |
+
# Create feature specs matching the config
|
| 62 |
+
from src.features.specs import ModalitySpec
|
| 63 |
+
|
| 64 |
+
# Determine which specs to use based on dimensions
|
| 65 |
+
audio_dim = feature_config.get("audio_dim", 39)
|
| 66 |
+
video_dim = feature_config.get("video_dim", 38)
|
| 67 |
+
text_dim = feature_config.get("text_dim", 384)
|
| 68 |
+
|
| 69 |
+
# For now, use standard specs (can be enhanced to support other combinations)
|
| 70 |
+
spec = FeatureSpecification(
|
| 71 |
+
audio_spec=ModalitySpec(name="audio", expected_dim=audio_dim, temporal=True),
|
| 72 |
+
video_spec=ModalitySpec(name="video", expected_dim=video_dim, temporal=True),
|
| 73 |
+
text_spec=ModalitySpec(name="text", expected_dim=text_dim, temporal=True),
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
# Removed strict assertions because PCA determines final dimensions dynamically
|
| 77 |
+
return spec
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def setup_preprocessors(config: dict, feature_spec: FeatureSpecification) -> dict:
|
| 81 |
+
"""Set up feature preprocessors (normalization, PCA)."""
|
| 82 |
+
preprocessors = {}
|
| 83 |
+
|
| 84 |
+
# Audio preprocessor
|
| 85 |
+
audio_prep = FeaturePreprocessor(
|
| 86 |
+
normalizer_type="standard",
|
| 87 |
+
apply_pca=True,
|
| 88 |
+
pca_components=0.80,
|
| 89 |
+
pca_whiten=True,
|
| 90 |
+
)
|
| 91 |
+
preprocessors["audio"] = audio_prep
|
| 92 |
+
|
| 93 |
+
# Video preprocessor
|
| 94 |
+
video_prep = FeaturePreprocessor(
|
| 95 |
+
normalizer_type="standard",
|
| 96 |
+
apply_pca=True,
|
| 97 |
+
pca_components=0.80,
|
| 98 |
+
pca_whiten=True,
|
| 99 |
+
)
|
| 100 |
+
preprocessors["video"] = video_prep
|
| 101 |
+
|
| 102 |
+
# Text preprocessor (Aggressive reduction)
|
| 103 |
+
text_prep = FeaturePreprocessor(
|
| 104 |
+
normalizer_type="standard",
|
| 105 |
+
apply_pca=True,
|
| 106 |
+
pca_components=0.80,
|
| 107 |
+
pca_whiten=True,
|
| 108 |
+
)
|
| 109 |
+
preprocessors["text"] = text_prep
|
| 110 |
+
|
| 111 |
+
return preprocessors
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _try_load_preprocessors(checkpoint_dir: Path, logger: logging.Logger) -> dict | None:
|
| 115 |
+
"""Try to load already-saved PCA preprocessors from the most recent checkpoint.
|
| 116 |
+
|
| 117 |
+
Reuses the PCA transform computed in a previous training run so that:
|
| 118 |
+
- dimensions stay consistent with the saved best_model.pt
|
| 119 |
+
- no refitting noise between runs
|
| 120 |
+
- faster startup
|
| 121 |
+
|
| 122 |
+
Returns None if no valid checkpoint exists.
|
| 123 |
+
"""
|
| 124 |
+
# Preference order: multimodal_v2 (best validated run), then any other
|
| 125 |
+
candidates = ["multimodal_v3", "multimodal_v2", "multimodal_v1"]
|
| 126 |
+
for name in candidates:
|
| 127 |
+
prep_dir = checkpoint_dir / name / "preprocessors"
|
| 128 |
+
if not prep_dir.exists():
|
| 129 |
+
continue
|
| 130 |
+
missing = [m for m in ["audio", "video", "text"] if not (prep_dir / f"{m}_preprocessor.pkl").exists()]
|
| 131 |
+
if missing:
|
| 132 |
+
continue
|
| 133 |
+
try:
|
| 134 |
+
from src.inference.validation import FeatureCheckpoint
|
| 135 |
+
manager = FeatureCheckpoint(checkpoint_dir)
|
| 136 |
+
preps = manager.load_preprocessors(name)
|
| 137 |
+
if all(preps.get(m) and preps[m].output_dim for m in ["audio", "video", "text"]):
|
| 138 |
+
logger.info(
|
| 139 |
+
f"Loaded existing PCA preprocessors from '{name}' checkpoint: "
|
| 140 |
+
f"audio={preps['audio'].output_dim}d "
|
| 141 |
+
f"video={preps['video'].output_dim}d "
|
| 142 |
+
f"text={preps['text'].output_dim}d"
|
| 143 |
+
)
|
| 144 |
+
return preps
|
| 145 |
+
except Exception as exc:
|
| 146 |
+
logger.warning(f"Could not load preprocessors from '{name}': {exc}")
|
| 147 |
+
return None
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def create_datasets_with_preprocessing(
|
| 151 |
+
config: dict,
|
| 152 |
+
preprocessors: dict,
|
| 153 |
+
feature_spec: FeatureSpecification,
|
| 154 |
+
logger: logging.Logger,
|
| 155 |
+
checkpoint_dir: Path | None = None,
|
| 156 |
+
) -> tuple:
|
| 157 |
+
"""Create training and validation datasets with preprocessing.
|
| 158 |
+
|
| 159 |
+
Tries to load already-saved PCA preprocessors first. Only refits if none exist.
|
| 160 |
+
"""
|
| 161 |
+
|
| 162 |
+
reg_cfg = config.get("regularization", {})
|
| 163 |
+
feat_noise = float(reg_cfg.get("feature_noise_std", 0.01))
|
| 164 |
+
temp_drop = float(reg_cfg.get("temporal_dropout_rate", 0.10))
|
| 165 |
+
|
| 166 |
+
train_dataset_raw = DatasetBuilder.build_from_csv(
|
| 167 |
+
split_csv_path=Path(config["dataset"]["train_split"]),
|
| 168 |
+
feature_dir=Path(config["dataset"]["feature_dir"]),
|
| 169 |
+
id_column=config["dataset"].get("id_column", "participant_id"),
|
| 170 |
+
label_column=config["dataset"].get("label_column", "phq_total"),
|
| 171 |
+
augment=True,
|
| 172 |
+
temporal_dropout_rate=temp_drop,
|
| 173 |
+
feature_noise_std=feat_noise,
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
val_dataset_raw = DatasetBuilder.build_from_csv(
|
| 177 |
+
split_csv_path=Path(config["dataset"]["val_split"]),
|
| 178 |
+
feature_dir=Path(config["dataset"]["feature_dir"]),
|
| 179 |
+
id_column=config["dataset"].get("id_column", "participant_id"),
|
| 180 |
+
label_column=config["dataset"].get("label_column", "phq_total"),
|
| 181 |
+
augment=False,
|
| 182 |
+
temporal_dropout_rate=0.0,
|
| 183 |
+
feature_noise_std=0.0,
|
| 184 |
+
)
|
| 185 |
+
|
| 186 |
+
logger.info(f"Loaded {len(train_dataset_raw)} training samples")
|
| 187 |
+
logger.info(f"Loaded {len(val_dataset_raw)} validation samples")
|
| 188 |
+
|
| 189 |
+
# Try to reuse already-saved PCA preprocessors (avoids refitting noise)
|
| 190 |
+
saved = _try_load_preprocessors(checkpoint_dir or Path("checkpoints"), logger)
|
| 191 |
+
if saved:
|
| 192 |
+
preprocessors = saved
|
| 193 |
+
logger.info("Using saved PCA preprocessors — skipping refit.")
|
| 194 |
+
else:
|
| 195 |
+
# Fit PCA preprocessors fresh on training data
|
| 196 |
+
logger.info("No saved preprocessors found. Fitting PCA from scratch...")
|
| 197 |
+
for modality in ["audio", "video", "text"]:
|
| 198 |
+
features_list = [
|
| 199 |
+
s.get(modality) for s in train_dataset_raw.samples
|
| 200 |
+
if s.get(modality) is not None
|
| 201 |
+
]
|
| 202 |
+
if features_list:
|
| 203 |
+
all_features = np.vstack(features_list)
|
| 204 |
+
preprocessors[modality].fit(all_features)
|
| 205 |
+
logger.info(
|
| 206 |
+
f" {modality}: PCA fitted → {preprocessors[modality].output_dim}d "
|
| 207 |
+
f"(from {all_features.shape[1]}d, {all_features.shape[0]} rows)"
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
# Apply preprocessors to both splits
|
| 211 |
+
logger.info("Applying PCA preprocessors to datasets...")
|
| 212 |
+
for split_name, dataset in [("train", train_dataset_raw), ("val", val_dataset_raw)]:
|
| 213 |
+
for sample in dataset.samples:
|
| 214 |
+
for modality in ["audio", "video", "text"]:
|
| 215 |
+
if sample.get(modality) is not None:
|
| 216 |
+
sample[modality] = preprocessors[modality].transform(sample[modality])
|
| 217 |
+
logger.info(
|
| 218 |
+
f" {split_name}: applied PCA — audio now {dataset.samples[0]['audio'].shape[1] if dataset.samples[0].get('audio') is not None else 'N/A'}d"
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
return train_dataset_raw, val_dataset_raw, preprocessors
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def main() -> None:
|
| 225 |
+
"""Main training function."""
|
| 226 |
+
parser = argparse.ArgumentParser(description="Train improved multimodal depression model")
|
| 227 |
+
parser.add_argument("--config", type=Path, default=Path("configs/training.yaml"))
|
| 228 |
+
parser.add_argument("--experiment-name", type=str, default="multimodal_v2")
|
| 229 |
+
parser.add_argument("--use-ema", action="store_true", help="Use exponential moving average")
|
| 230 |
+
parser.add_argument("--seed", type=int, default=42, help="Random seed for reproducibility")
|
| 231 |
+
args = parser.parse_args()
|
| 232 |
+
|
| 233 |
+
# Fix random seed for reproducibility
|
| 234 |
+
import random
|
| 235 |
+
random.seed(args.seed)
|
| 236 |
+
np.random.seed(args.seed)
|
| 237 |
+
torch.manual_seed(args.seed)
|
| 238 |
+
if torch.cuda.is_available():
|
| 239 |
+
torch.cuda.manual_seed_all(args.seed)
|
| 240 |
+
|
| 241 |
+
# Load config
|
| 242 |
+
config = load_config(args.config)
|
| 243 |
+
|
| 244 |
+
# Set up logging
|
| 245 |
+
run_dir = configure_logging(
|
| 246 |
+
Path(config.get("logging", {}).get("log_dir", "logs")),
|
| 247 |
+
experiment_name=args.experiment_name,
|
| 248 |
+
)
|
| 249 |
+
logger = logging.getLogger(__name__)
|
| 250 |
+
|
| 251 |
+
logger.info(f"Starting training: {args.experiment_name}")
|
| 252 |
+
logger.info(f"Config: {args.config}")
|
| 253 |
+
|
| 254 |
+
# Feature specification and preprocessing
|
| 255 |
+
logger.info("Setting up feature pipeline...")
|
| 256 |
+
feature_spec = setup_feature_specs(config)
|
| 257 |
+
logger.info(f"Feature spec - Audio: {feature_spec.audio_dim}D, Video: {feature_spec.video_dim}D, Text: {feature_spec.text_dim}D")
|
| 258 |
+
|
| 259 |
+
preprocessors = setup_preprocessors(config, feature_spec)
|
| 260 |
+
|
| 261 |
+
checkpoint_dir = Path(config["training"].get("checkpoint_dir", "checkpoints"))
|
| 262 |
+
|
| 263 |
+
# Create datasets — loads existing PCA preprocessors if available
|
| 264 |
+
train_dataset, val_dataset, preprocessors = create_datasets_with_preprocessing(
|
| 265 |
+
config,
|
| 266 |
+
preprocessors,
|
| 267 |
+
feature_spec,
|
| 268 |
+
logger,
|
| 269 |
+
checkpoint_dir=checkpoint_dir,
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
# Update model config dynamically based on PCA output dims
|
| 273 |
+
if preprocessors["audio"].output_dim:
|
| 274 |
+
config["model"]["audio_dim"] = preprocessors["audio"].output_dim
|
| 275 |
+
if preprocessors["video"].output_dim:
|
| 276 |
+
config["model"]["video_dim"] = preprocessors["video"].output_dim
|
| 277 |
+
if preprocessors["text"].output_dim:
|
| 278 |
+
config["model"]["text_dim"] = preprocessors["text"].output_dim
|
| 279 |
+
|
| 280 |
+
logger.info(f"Final model dims - Audio: {config['model']['audio_dim']}, Video: {config['model']['video_dim']}, Text: {config['model']['text_dim']}")
|
| 281 |
+
|
| 282 |
+
# Create dataloaders
|
| 283 |
+
train_loader = DataLoader(
|
| 284 |
+
train_dataset,
|
| 285 |
+
batch_size=int(config["training"].get("batch_size", 8)),
|
| 286 |
+
shuffle=True,
|
| 287 |
+
num_workers=int(config["training"].get("num_workers", 2)),
|
| 288 |
+
collate_fn=multimodal_collate_fn,
|
| 289 |
+
)
|
| 290 |
+
val_loader = DataLoader(
|
| 291 |
+
val_dataset,
|
| 292 |
+
batch_size=int(config["training"].get("batch_size", 8)),
|
| 293 |
+
shuffle=False,
|
| 294 |
+
num_workers=int(config["training"].get("num_workers", 2)),
|
| 295 |
+
collate_fn=multimodal_collate_fn,
|
| 296 |
+
)
|
| 297 |
+
|
| 298 |
+
# Model
|
| 299 |
+
model_config = config.get("model", {})
|
| 300 |
+
model = MultimodalDepressionModel(**model_config)
|
| 301 |
+
logger.info(f"Model parameters: {sum(p.numel() for p in model.parameters()):,}")
|
| 302 |
+
|
| 303 |
+
# Optimizer and scheduler
|
| 304 |
+
optimizer = torch.optim.AdamW(
|
| 305 |
+
model.parameters(),
|
| 306 |
+
lr=float(config["training"].get("learning_rate", 1e-4)),
|
| 307 |
+
weight_decay=float(config["training"].get("weight_decay", 1e-4)),
|
| 308 |
+
)
|
| 309 |
+
|
| 310 |
+
# Warmup scheduler
|
| 311 |
+
warmup_scheduler = WarmupScheduler(
|
| 312 |
+
optimizer,
|
| 313 |
+
warmup_epochs=5,
|
| 314 |
+
total_epochs=int(config["training"].get("max_epochs", 100)),
|
| 315 |
+
base_lr=float(config["training"].get("learning_rate", 1e-4)),
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
# Reduce on plateau scheduler
|
| 319 |
+
plateau_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
|
| 320 |
+
optimizer,
|
| 321 |
+
mode="min",
|
| 322 |
+
factor=float(config["training"].get("lr_factor", 0.5)),
|
| 323 |
+
patience=int(config["training"].get("lr_patience", 5)),
|
| 324 |
+
)
|
| 325 |
+
|
| 326 |
+
# EMA if enabled
|
| 327 |
+
ema = None
|
| 328 |
+
if args.use_ema:
|
| 329 |
+
ema = EMA(model, decay=0.999)
|
| 330 |
+
logger.info("Using exponential moving average")
|
| 331 |
+
|
| 332 |
+
# Merge loss weights into training config for Trainer
|
| 333 |
+
training_config = config["training"].copy()
|
| 334 |
+
if "loss" in config:
|
| 335 |
+
training_config.update(config["loss"])
|
| 336 |
+
|
| 337 |
+
# Trainer
|
| 338 |
+
trainer = Trainer(
|
| 339 |
+
model=model,
|
| 340 |
+
train_loader=train_loader,
|
| 341 |
+
val_loader=val_loader,
|
| 342 |
+
optimizer=optimizer,
|
| 343 |
+
scheduler=plateau_scheduler,
|
| 344 |
+
config=training_config,
|
| 345 |
+
)
|
| 346 |
+
|
| 347 |
+
# Training
|
| 348 |
+
logger.info("Starting training...")
|
| 349 |
+
history = trainer.train()
|
| 350 |
+
|
| 351 |
+
# Save best model info
|
| 352 |
+
best_loss = getattr(trainer, "best_loss", None)
|
| 353 |
+
best_ccc = getattr(trainer, "best_ccc", None)
|
| 354 |
+
logger.info("Training complete!")
|
| 355 |
+
logger.info(f"Best CCC: {best_ccc:.4f}" if best_ccc else "Best CCC: N/A")
|
| 356 |
+
|
| 357 |
+
# Write result.json
|
| 358 |
+
result_data = {
|
| 359 |
+
"best_ccc": float(best_ccc) if best_ccc else 0.0,
|
| 360 |
+
"best_loss": float(best_loss) if best_loss else 0.0,
|
| 361 |
+
"experiment_name": args.experiment_name,
|
| 362 |
+
"config": convert_numpy(config),
|
| 363 |
+
"run_dir": str(run_dir),
|
| 364 |
+
}
|
| 365 |
+
import json
|
| 366 |
+
with open("result.json", "w") as f:
|
| 367 |
+
json.dump(result_data, f, indent=4)
|
| 368 |
+
logger.info(f"Best loss: {history.get('best_loss', 'N/A')}")
|
| 369 |
+
|
| 370 |
+
# Save preprocessors for inference
|
| 371 |
+
checkpoint_manager = FeatureCheckpoint(Path(config["training"].get("checkpoint_dir", "checkpoints")))
|
| 372 |
+
checkpoint_manager.save_preprocessors(preprocessors, args.experiment_name)
|
| 373 |
+
checkpoint_manager.save_feature_spec(feature_spec, args.experiment_name)
|
| 374 |
+
logger.info(f"Saved preprocessors and feature spec to {args.experiment_name}")
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
if __name__ == "__main__":
|
| 378 |
+
main()
|
src/__init__.py
ADDED
|
File without changes
|
src/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (120 Bytes). View file
|
|
|
src/api/__init__.py
ADDED
|
File without changes
|
src/api/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (124 Bytes). View file
|
|
|
src/api/__pycache__/app.cpython-312.pyc
ADDED
|
Binary file (5.05 kB). View file
|
|
|
src/api/__pycache__/routes.cpython-312.pyc
ADDED
|
Binary file (14 kB). View file
|
|
|