koushik2109 commited on
Commit
8d7b4b1
·
1 Parent(s): 319bd68

Model Changes

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .dockerignore +28 -0
  2. Dockerfile +32 -0
  3. checkpoints/best_fusion.pt +3 -0
  4. checkpoints/best_model.pt +3 -0
  5. checkpoints/best_model_backup_0.14ccc.pt +3 -0
  6. checkpoints/best_stage1.pt +3 -0
  7. checkpoints/best_stage2_clean.pt +3 -0
  8. checkpoints/best_trimodal.pt +3 -0
  9. checkpoints/best_trimodal_backup.pt +3 -0
  10. checkpoints/best_trimodal_text.pt +3 -0
  11. checkpoints/cv/best_model_fold0.pt +3 -0
  12. checkpoints/cv/best_model_fold1.pt +3 -0
  13. checkpoints/cv/best_model_fold2.pt +3 -0
  14. checkpoints/cv/best_model_fold3.pt +3 -0
  15. checkpoints/cv/best_model_fold4.pt +3 -0
  16. checkpoints/multimodal_v2/feature_spec.json +32 -0
  17. checkpoints/multimodal_v2/preprocessors/audio_preprocessor.pkl +3 -0
  18. checkpoints/multimodal_v2/preprocessors/text_preprocessor.pkl +3 -0
  19. checkpoints/multimodal_v2/preprocessors/video_preprocessor.pkl +3 -0
  20. checkpoints/multimodal_v3/feature_spec.json +32 -0
  21. checkpoints/multimodal_v3/preprocessors/audio_preprocessor.pkl +3 -0
  22. checkpoints/multimodal_v3/preprocessors/text_preprocessor.pkl +3 -0
  23. checkpoints/multimodal_v3/preprocessors/video_preprocessor.pkl +3 -0
  24. checkpoints/multimodal_v4/feature_spec.json +32 -0
  25. checkpoints/multimodal_v4/preprocessors/audio_preprocessor.pkl +3 -0
  26. checkpoints/multimodal_v4/preprocessors/text_preprocessor.pkl +3 -0
  27. checkpoints/multimodal_v4/preprocessors/video_preprocessor.pkl +3 -0
  28. checkpoints/pca_cnn_transform.pkl +3 -0
  29. checkpoints/scalers/feature_scalers.pkl +3 -0
  30. checkpoints/scalers/pca_reducer.pkl +3 -0
  31. configs/feature_extraction.yaml +17 -0
  32. configs/inference.yaml +19 -0
  33. configs/training.yaml +55 -0
  34. requirements.txt +26 -0
  35. scripts/__pycache__/train.cpython-312.pyc +0 -0
  36. scripts/__pycache__/train_v2.cpython-312.pyc +0 -0
  37. scripts/auto_train.py +51 -0
  38. scripts/evaluate.py +68 -0
  39. scripts/extract_features.py +54 -0
  40. scripts/infer_live_v2.py +96 -0
  41. scripts/serve.py +37 -0
  42. scripts/smoke_infer_test.py +37 -0
  43. scripts/train.py +93 -0
  44. scripts/train_v2.py +378 -0
  45. src/__init__.py +0 -0
  46. src/__pycache__/__init__.cpython-312.pyc +0 -0
  47. src/api/__init__.py +0 -0
  48. src/api/__pycache__/__init__.cpython-312.pyc +0 -0
  49. src/api/__pycache__/app.cpython-312.pyc +0 -0
  50. src/api/__pycache__/routes.cpython-312.pyc +0 -0
.dockerignore ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ .venv/
2
+ .git/
3
+ __pycache__/
4
+ *.pyc
5
+ *.pyo
6
+ logs*/
7
+ logs_v4_run.log
8
+ data/
9
+ notebooks/
10
+ docs/
11
+ runs/
12
+ *.zip
13
+ *.tar.gz
14
+ checkpoints/cv/
15
+ checkpoints/multimodal_v2/
16
+ checkpoints/multimodal_v3/
17
+ checkpoints/best_fusion.pt
18
+ checkpoints/best_model_backup_0.14ccc.pt
19
+ checkpoints/best_stage1.pt
20
+ checkpoints/best_stage2_clean.pt
21
+ checkpoints/best_trimodal.pt
22
+ checkpoints/best_trimodal_backup.pt
23
+ checkpoints/best_trimodal_text.pt
24
+ checkpoints/pca_cnn_transform.pkl
25
+ checkpoints/scalers/
26
+ windows/
27
+ .vscode/
28
+ tests/
Dockerfile ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.11-slim
2
+
3
+ WORKDIR /app
4
+
5
+ # System deps: libgl (mediapipe/opencv), ffmpeg (audio), libgomp (torch)
6
+ RUN apt-get update && apt-get install -y --no-install-recommends \
7
+ libgl1 \
8
+ libglib2.0-0 \
9
+ libsm6 \
10
+ libxext6 \
11
+ libxrender-dev \
12
+ libgomp1 \
13
+ ffmpeg \
14
+ git \
15
+ && rm -rf /var/lib/apt/lists/*
16
+
17
+ COPY requirements.txt .
18
+
19
+ # Install CPU-only torch — latest from wheel index (no version pin)
20
+ # Same approach as start-all.sh: installs latest compatible cpu set
21
+ RUN pip install --no-cache-dir \
22
+ torch torchaudio torchvision \
23
+ --index-url https://download.pytorch.org/whl/cpu
24
+
25
+ # Install remaining deps (torch already satisfied above, won't re-download)
26
+ RUN pip install --no-cache-dir -r requirements.txt
27
+
28
+ COPY . .
29
+
30
+ EXPOSE 8001
31
+
32
+ CMD ["python", "scripts/serve.py", "--port", "8001"]
checkpoints/best_fusion.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a3f27f622fcf180f1704d4612273a97813a85d8893b87e9a185d2915cdcbeda
3
+ size 43669
checkpoints/best_model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:147924ec16e02220048ff2c28ad71daafe3a93d35771ca9e4e4de1279c4e78c3
3
+ size 405941
checkpoints/best_model_backup_0.14ccc.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f0ed9844a410329a275b9fc7bbdc3e89128ebeb292e568ae6695d329936177d
3
+ size 43125
checkpoints/best_stage1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc5946d5372483f4eaeadba08255ce16eaa7bf656b7ca15a0c5a8d12d1ea3551
3
+ size 349073
checkpoints/best_stage2_clean.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b966db1077d73ce7e88a8e212b3d16c3f5994d8d128007e69f944f6adf82c505
3
+ size 1870025
checkpoints/best_trimodal.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c50f63feb8adcce165846a833ac72bf6601c2d96624b24b71d12ced38627b6f
3
+ size 1869605
checkpoints/best_trimodal_backup.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:39ab55d17b81e93f47cbf988a190fd26e8f6a83570f366321a4daf1835f4d046
3
+ size 1869255
checkpoints/best_trimodal_text.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:818cccc4fab056005b651ead9ef7bb5c430be39b31a291d3928a9002e9813405
3
+ size 1891597
checkpoints/cv/best_model_fold0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2783e75575e344833fbc9224a7191e2be957cdc6d6f3016378618b11ac6bc930
3
+ size 43339
checkpoints/cv/best_model_fold1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a1b18a628524c2aa70398976ed90d36131ffbe7910675a5792f17540f244ea77
3
+ size 43339
checkpoints/cv/best_model_fold2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f616da338d0dadba40884826c4959d47afeae3a321ffc907445fa6afcb8d7e4
3
+ size 43339
checkpoints/cv/best_model_fold3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:937ba7d3640334ef41b05f15bc4bc2581ba60f8fe3711ff9fe2f406154db05f7
3
+ size 43339
checkpoints/cv/best_model_fold4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44f2863172d498f05bd4b34fb7fa56b131cc74e9cf2c986d78537acc85fdbaf6
3
+ size 43339
checkpoints/multimodal_v2/feature_spec.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "audio": {
3
+ "name": "audio",
4
+ "expected_dim": 62,
5
+ "temporal": true,
6
+ "preprocessor_class": null,
7
+ "extractor_class": null,
8
+ "pca_enabled": false,
9
+ "pca_target_dim": null,
10
+ "normalizer_type": "standard"
11
+ },
12
+ "video": {
13
+ "name": "video",
14
+ "expected_dim": 49,
15
+ "temporal": true,
16
+ "preprocessor_class": null,
17
+ "extractor_class": null,
18
+ "pca_enabled": false,
19
+ "pca_target_dim": null,
20
+ "normalizer_type": "standard"
21
+ },
22
+ "text": {
23
+ "name": "text",
24
+ "expected_dim": 384,
25
+ "temporal": true,
26
+ "preprocessor_class": null,
27
+ "extractor_class": null,
28
+ "pca_enabled": false,
29
+ "pca_target_dim": null,
30
+ "normalizer_type": "standard"
31
+ }
32
+ }
checkpoints/multimodal_v2/preprocessors/audio_preprocessor.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc6356119bc5f47417ee3f5fd57585a9c83e4258a83aa223c4ab34f73714b2af
3
+ size 11640
checkpoints/multimodal_v2/preprocessors/text_preprocessor.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ed802752135f85d0d847b33077e36492a27511ad35df1ca5cfcfa6b9a6af0226
3
+ size 264428
checkpoints/multimodal_v2/preprocessors/video_preprocessor.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06b856001f26ddfe1b74400f476d6c2b80e000b67bd571f29e4b8857cd129df4
3
+ size 11016
checkpoints/multimodal_v3/feature_spec.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "audio": {
3
+ "name": "audio",
4
+ "expected_dim": 62,
5
+ "temporal": true,
6
+ "preprocessor_class": null,
7
+ "extractor_class": null,
8
+ "pca_enabled": false,
9
+ "pca_target_dim": null,
10
+ "normalizer_type": "standard"
11
+ },
12
+ "video": {
13
+ "name": "video",
14
+ "expected_dim": 49,
15
+ "temporal": true,
16
+ "preprocessor_class": null,
17
+ "extractor_class": null,
18
+ "pca_enabled": false,
19
+ "pca_target_dim": null,
20
+ "normalizer_type": "standard"
21
+ },
22
+ "text": {
23
+ "name": "text",
24
+ "expected_dim": 384,
25
+ "temporal": true,
26
+ "preprocessor_class": null,
27
+ "extractor_class": null,
28
+ "pca_enabled": false,
29
+ "pca_target_dim": null,
30
+ "normalizer_type": "standard"
31
+ }
32
+ }
checkpoints/multimodal_v3/preprocessors/audio_preprocessor.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:caff65d8a1812be62c39d70901e1801791631910cfda40e08443a04ae8efa30d
3
+ size 11733
checkpoints/multimodal_v3/preprocessors/text_preprocessor.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5ca025ec6356361869c37c00b5b5a4e5bf4b0307192801010889fc67ea495c8a
3
+ size 264521
checkpoints/multimodal_v3/preprocessors/video_preprocessor.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:963825c9464b72a6986599744b34fad3303529b2040773cfc2470e675d0650dd
3
+ size 11109
checkpoints/multimodal_v4/feature_spec.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "audio": {
3
+ "name": "audio",
4
+ "expected_dim": 62,
5
+ "temporal": true,
6
+ "preprocessor_class": null,
7
+ "extractor_class": null,
8
+ "pca_enabled": false,
9
+ "pca_target_dim": null,
10
+ "normalizer_type": "standard"
11
+ },
12
+ "video": {
13
+ "name": "video",
14
+ "expected_dim": 49,
15
+ "temporal": true,
16
+ "preprocessor_class": null,
17
+ "extractor_class": null,
18
+ "pca_enabled": false,
19
+ "pca_target_dim": null,
20
+ "normalizer_type": "standard"
21
+ },
22
+ "text": {
23
+ "name": "text",
24
+ "expected_dim": 384,
25
+ "temporal": true,
26
+ "preprocessor_class": null,
27
+ "extractor_class": null,
28
+ "pca_enabled": false,
29
+ "pca_target_dim": null,
30
+ "normalizer_type": "standard"
31
+ }
32
+ }
checkpoints/multimodal_v4/preprocessors/audio_preprocessor.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:caff65d8a1812be62c39d70901e1801791631910cfda40e08443a04ae8efa30d
3
+ size 11733
checkpoints/multimodal_v4/preprocessors/text_preprocessor.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5ca025ec6356361869c37c00b5b5a4e5bf4b0307192801010889fc67ea495c8a
3
+ size 264521
checkpoints/multimodal_v4/preprocessors/video_preprocessor.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:963825c9464b72a6986599744b34fad3303529b2040773cfc2470e675d0650dd
3
+ size 11109
checkpoints/pca_cnn_transform.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:654fe52593b0a178def46f0cb3479dd4558e157233b9a7f3ed031df0a98b131d
3
+ size 1059084
checkpoints/scalers/feature_scalers.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:878445280c1c0c5d1799346f07d08d0e915d27ed4b1d2ed1663c99ca440250d1
3
+ size 1942
checkpoints/scalers/pca_reducer.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17451fb4ae35aeb65c907cc3dd78ca2663f8f57473675e55d1d253e1e2300227
3
+ size 120065
configs/feature_extraction.yaml ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ audio:
2
+ sample_rate: 16000
3
+ frame_rate: 8
4
+ chunk_duration: 2.0
5
+ chunk_overlap: 0.5
6
+ mfcc_n_mfcc: 13
7
+ hubert_model_name: facebook/hubert-base-ls960
8
+ feature_backends:
9
+ - mfcc
10
+ - hubert
11
+ video:
12
+ frame_rate: 8
13
+ temporal_window: 32
14
+ openface_csv_key: openface
15
+ embedding_backend: resnet50
16
+ text:
17
+ transformer_model: sentence-transformers/all-mpnet-base-v2
configs/inference.yaml ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ artifacts:
2
+ checkpoint_path: checkpoints/best_model.pt
3
+ model:
4
+ audio_dim: 33
5
+ video_dim: 40
6
+ text_dim: 163
7
+ fusion_dim: 32
8
+ num_questions: 8
9
+ fusion_mode: hybrid
10
+ encoder_type: conv_gru
11
+ num_layers: 1
12
+ dropout: 0.1
13
+ inference:
14
+ device: auto
15
+ batch_size: 4
16
+ api:
17
+ port: 8000
18
+ logging:
19
+ log_dir: logs
configs/training.yaml ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset:
2
+ feature_dir: data/npz
3
+ id_column: participant_id
4
+ label_column: phq_score
5
+ train_split: data/splits/train_split.csv
6
+ val_split: data/splits/dev_split.csv
7
+ features:
8
+ audio_dim: 62
9
+ text_dim: 384
10
+ video_dim: 49
11
+ logging:
12
+ experiment_name: multimodal_v4
13
+ log_dir: logs_v4
14
+ loss:
15
+ # Keep exact weights from multimodal_v3 that achieved CCC=0.5058
16
+ ccc_weight: 30.0
17
+ regression_weight: 0.3
18
+ question_weight: 0.25
19
+ classification_weight: 0.05
20
+ confidence_weight: 0.01
21
+ entropy_weight: 0.2
22
+ # Zero gate_balance → text can freely exceed 1/3 without competing loss
23
+ gate_balance_weight: 0.0
24
+ # Text gate preference — penalise when text gate < 0.40
25
+ text_gate_weight: 0.25
26
+ text_gate_target: 0.40
27
+ model:
28
+ # Exact dims from multimodal_v3 (proven CCC=0.5058, 91K params)
29
+ audio_dim: 33
30
+ video_dim: 40
31
+ text_dim: 163
32
+ dropout: 0.45
33
+ encoder_type: conv_gru
34
+ fusion_dim: 32
35
+ fusion_mode: hybrid
36
+ modality_dropout: 0.20
37
+ num_heads: 4
38
+ num_layers: 1
39
+ num_questions: 8
40
+ regularization:
41
+ feature_noise_std: 0.06
42
+ temporal_dropout_rate: 0.20
43
+ training:
44
+ batch_size: 16
45
+ early_stopping_min_delta: 0.001
46
+ early_stopping_patience: 200
47
+ grad_clip_norm: 1.0
48
+ learning_rate: 0.001
49
+ lr_factor: 0.5
50
+ lr_patience: 30
51
+ max_epochs: 1000
52
+ num_workers: 0
53
+ use_amp: true
54
+ weight_decay: 8e-3
55
+ mixup_alpha: 0.3
requirements.txt ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ torch>=2.1.2
2
+ torchaudio>=2.1.2
3
+ torchvision>=0.16.2
4
+ numpy>=1.24.4
5
+ scikit-learn>=1.3.2
6
+ pandas>=2.0.3
7
+ librosa>=0.10.1
8
+ soundfile>=0.12.1
9
+ pydub>=0.25.1
10
+ opensmile>=2.4.2
11
+ transformers>=4.36.2,<5.6
12
+ sentence-transformers>=2.2.2,<5.5
13
+ fastapi>=0.104.1
14
+ uvicorn>=0.24.0
15
+ slowapi>=0.1.9
16
+ python-dotenv>=1.0.0
17
+ pyyaml>=6.0.1
18
+ sqlalchemy>=2.0.23
19
+ tqdm>=4.66.1
20
+ joblib>=1.3.2
21
+ pydantic>=2.5.3
22
+ requests>=2.31.0
23
+ python-multipart>=0.0.6
24
+ scipy>=1.11.4
25
+ h5py>=3.10.0
26
+ matplotlib>=3.8.2
scripts/__pycache__/train.cpython-312.pyc ADDED
Binary file (4.97 kB). View file
 
scripts/__pycache__/train_v2.cpython-312.pyc ADDED
Binary file (17 kB). View file
 
scripts/auto_train.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import subprocess
3
+ import yaml
4
+
5
+ config_path = "configs/training.yaml"
6
+
7
+ def run_training():
8
+ print("Running training...")
9
+ result = subprocess.run(["python3", "scripts/train_v2.py", "--config", config_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
10
+
11
+ best_ccc = -1.0
12
+ for line in result.stdout.split('\n'):
13
+ if "--> New best model saved with CCC:" in line:
14
+ try:
15
+ ccc = float(line.split(":")[-1].strip())
16
+ if ccc > best_ccc:
17
+ best_ccc = ccc
18
+ except:
19
+ pass
20
+ print(f"Training finished. Best CCC: {best_ccc}")
21
+ if best_ccc == -1.0:
22
+ print("Training failed or did not report CCC. Output:")
23
+ print(result.stdout)
24
+ return best_ccc
25
+
26
+ def update_config(lr, dropout, ccc_weight):
27
+ with open(config_path, "r") as f:
28
+ config = yaml.safe_load(f)
29
+
30
+ config["model"]["dropout"] = dropout
31
+ config["training"]["learning_rate"] = lr
32
+ config["loss"]["ccc_weight"] = ccc_weight
33
+
34
+ with open(config_path, "w") as f:
35
+ yaml.safe_dump(config, f)
36
+
37
+ lrs = [5e-4, 1e-4, 8e-4]
38
+ dropouts = [0.1, 0.2, 0.3]
39
+ ccc_weights = [1.5, 2.0, 3.0]
40
+
41
+ for lr in lrs:
42
+ for drop in dropouts:
43
+ for cw in ccc_weights:
44
+ print(f"Testing LR: {lr}, Dropout: {drop}, CCC Weight: {cw}")
45
+ update_config(lr, drop, cw)
46
+ ccc = run_training()
47
+ if ccc >= 0.55:
48
+ print(f"Success! Reached {ccc} CCC with LR={lr}, Drop={drop}, CCC Weight={cw}")
49
+ exit(0)
50
+
51
+ print("Failed to reach 0.55 CCC after all trials.")
scripts/evaluate.py ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import logging
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ ROOT_DIR = Path(__file__).resolve().parents[1]
7
+ if str(ROOT_DIR) not in sys.path:
8
+ sys.path.insert(0, str(ROOT_DIR))
9
+
10
+ import yaml
11
+ import torch
12
+ from torch.utils.data import DataLoader
13
+
14
+ from src.dataset.builder import DatasetBuilder
15
+ from src.dataset.collate import multimodal_collate_fn
16
+ from src.models.multimodal_model import MultimodalDepressionModel
17
+ from src.training.trainer import Trainer
18
+ from src.utils.logging import configure_logging
19
+ from src.evaluation.evaluator import evaluate_predictions
20
+
21
+
22
+ def load_config(path: Path) -> dict:
23
+ with open(path, "r", encoding="utf-8") as handle:
24
+ return yaml.safe_load(handle)
25
+
26
+
27
+ def main() -> None:
28
+ parser = argparse.ArgumentParser(description="Evaluate ModelV2 checkpoint")
29
+ parser.add_argument("--config", type=Path, default=Path("configs/inference.yaml"))
30
+ parser.add_argument("--checkpoint", type=Path, required=True)
31
+ args = parser.parse_args()
32
+
33
+ config = load_config(args.config)
34
+ configure_logging(Path(config.get("logging", {}).get("log_dir", "logs")))
35
+ logger = logging.getLogger(__name__)
36
+
37
+ val_dataset = DatasetBuilder.build_from_csv(
38
+ split_csv_path=Path(config["dataset"]["val_split"]),
39
+ feature_dir=Path(config["dataset"]["feature_dir"]),
40
+ id_column=config["dataset"].get("id_column", "participant_id"),
41
+ label_column=config["dataset"].get("label_column", "phq_total"),
42
+ augment=False,
43
+ )
44
+ val_loader = DataLoader(val_dataset, batch_size=config.get("inference", {}).get("batch_size", 4), shuffle=False, collate_fn=multimodal_collate_fn)
45
+
46
+ model = MultimodalDepressionModel(**config["model"])
47
+ model.load_state_dict(torch.load(args.checkpoint, map_location="cpu"))
48
+ model.eval()
49
+
50
+ y_true, y_pred, q_true, q_pred, cls_true, cls_pred = [], [], [], [], [], []
51
+ with torch.no_grad():
52
+ for batch in val_loader:
53
+ output = model(
54
+ batch["audio"], batch["video"], batch["text"], batch["audio_mask"], batch["video_mask"], batch["text_mask"],
55
+ )
56
+ y_true.extend(batch["phq_total"].tolist())
57
+ y_pred.extend(output["phq_total"].tolist())
58
+ q_true.extend(batch["phq_questions"].tolist())
59
+ q_pred.extend(output["phq_questions"].tolist())
60
+ cls_true.extend(batch["classification"].tolist())
61
+ cls_pred.extend(torch.sigmoid(output["classification"]).tolist())
62
+
63
+ results = evaluate_predictions(y_true, y_pred, q_true, q_pred, cls_true, cls_pred)
64
+ logger.info(f"Evaluation results: {results}")
65
+
66
+
67
+ if __name__ == "__main__":
68
+ main()
scripts/extract_features.py ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ from pathlib import Path
3
+ from typing import Dict, Any
4
+
5
+ import yaml
6
+
7
+ from src.features.audio_features import AudioFeatureExtractor
8
+ from src.features.video_features import VideoFeatureExtractor
9
+ from src.features.text_features import TextFeatureExtractor
10
+ from src.features.feature_store import save_features
11
+ from src.preprocessing.audio_preprocessor import AudioPreprocessor
12
+ from src.preprocessing.text_preprocessor import TextPreprocessor
13
+
14
+
15
+ def load_config(path: Path) -> Dict[str, Any]:
16
+ with open(path, "r", encoding="utf-8") as handle:
17
+ return yaml.safe_load(handle)
18
+
19
+
20
+ def main() -> None:
21
+ parser = argparse.ArgumentParser(description="Extract ModelV2 multimodal features")
22
+ parser.add_argument("--config", type=Path, default=Path("configs/feature_extraction.yaml"))
23
+ parser.add_argument("--input-dir", type=Path, required=True)
24
+ parser.add_argument("--output-dir", type=Path, required=True)
25
+ args = parser.parse_args()
26
+
27
+ config = load_config(args.config)
28
+ audio_preprocessor = AudioPreprocessor(**config["audio"])
29
+ audio_extractor = AudioFeatureExtractor(config["audio"])
30
+ video_extractor = VideoFeatureExtractor(config["video"])
31
+ text_preprocessor = TextPreprocessor()
32
+ text_extractor = TextFeatureExtractor(config["text"]["transformer_model"])
33
+
34
+ for subject_dir in args.input_dir.iterdir():
35
+ if not subject_dir.is_dir():
36
+ continue
37
+ participant_id = subject_dir.name
38
+ audio_path = subject_dir / "audio.wav"
39
+ transcript_path = subject_dir / "transcript.txt"
40
+ openface_path = subject_dir / "openface.csv"
41
+ if not audio_path.exists() or not transcript_path.exists() or not openface_path.exists():
42
+ continue
43
+ waveform = audio_preprocessor.load_audio(audio_path)
44
+ audio_chunks = audio_preprocessor.preprocess(waveform)
45
+ audio_features = {"audio": audio_extractor.extract_modern(waveform)}
46
+ video_behavior = video_extractor.extract_openface(openface_path)["behavior"]
47
+ text = transcript_path.read_text(encoding="utf-8")
48
+ chunks = text_preprocessor.preprocess(text)
49
+ text_features = {"text": text_extractor.encode(chunks)}
50
+ save_features(args.output_dir / f"{participant_id}.npz", {
51
+ **audio_features,
52
+ "video": video_behavior,
53
+ **text_features,
54
+ })
scripts/infer_live_v2.py ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Example live inference script for multimodal depression detection.
3
+
4
+ Demonstrates how to use the LiveInferencePipeline for real-time predictions
5
+ from webcam/video/audio/transcript inputs.
6
+ """
7
+
8
+ import argparse
9
+ import sys
10
+ from pathlib import Path
11
+
12
+ ROOT_DIR = Path(__file__).resolve().parents[2]
13
+ if str(ROOT_DIR) not in sys.path:
14
+ sys.path.insert(0, str(ROOT_DIR))
15
+
16
+ import torch
17
+ from src.inference.live_pipeline import LiveInferencePipeline
18
+ from src.features.specs import FeatureSpecification
19
+ from src.inference.validation import FeatureCheckpoint
20
+
21
+
22
+ def main() -> None:
23
+ """Example usage of live inference pipeline."""
24
+ parser = argparse.ArgumentParser(description="Live inference example")
25
+ parser.add_argument("--checkpoint", type=Path, required=True, help="Model checkpoint path")
26
+ parser.add_argument("--experiment", type=str, required=True, help="Experiment name for loading preprocessors")
27
+ parser.add_argument("--audio", type=Path, help="Path to audio file")
28
+ parser.add_argument("--openface-csv", type=Path, help="Path to OpenFace CSV (video features)")
29
+ parser.add_argument("--transcript", type=str, help="Transcript text")
30
+ parser.add_argument("--device", type=str, default="auto", help="Device (auto/cuda/cpu)")
31
+ parser.add_argument("--verbose", action="store_true", help="Verbose output")
32
+ args = parser.parse_args()
33
+
34
+ # Load feature spec and preprocessors
35
+ checkpoint_manager = FeatureCheckpoint(Path("checkpoints"))
36
+
37
+ try:
38
+ feature_spec = checkpoint_manager.load_feature_spec(args.experiment)
39
+ preprocessors = checkpoint_manager.load_preprocessors(args.experiment)
40
+ except FileNotFoundError as e:
41
+ print(f"Error: {e}")
42
+ print(f"Make sure to run training script with --experiment-name {args.experiment}")
43
+ sys.exit(1)
44
+
45
+ # Model config (same as training)
46
+ model_config = {
47
+ "audio_dim": feature_spec.audio_dim,
48
+ "video_dim": feature_spec.video_dim,
49
+ "text_dim": feature_spec.text_dim,
50
+ "fusion_dim": 256,
51
+ "num_questions": 8,
52
+ "encoder_type": "transformer",
53
+ "num_heads": 4,
54
+ "num_layers": 2,
55
+ "dropout": 0.2,
56
+ "fusion_mode": "hybrid",
57
+ }
58
+
59
+ # Initialize pipeline
60
+ pipeline = LiveInferencePipeline(
61
+ checkpoint_path=args.checkpoint,
62
+ model_config=model_config,
63
+ feature_spec=feature_spec,
64
+ preprocessors=preprocessors,
65
+ device=args.device,
66
+ )
67
+
68
+ if args.verbose:
69
+ print(f"Model loaded from: {args.checkpoint}")
70
+ print(f"Feature spec: audio={feature_spec.audio_dim}D, video={feature_spec.video_dim}D, text={feature_spec.text_dim}D")
71
+
72
+ # Run inference
73
+ try:
74
+ result = pipeline.predict_single(
75
+ audio_path=args.audio,
76
+ video_openface_csv=args.openface_csv,
77
+ transcript=args.transcript,
78
+ verbose=args.verbose,
79
+ )
80
+
81
+ # Print results
82
+ print(pipeline.predict_and_explain(
83
+ audio_path=args.audio,
84
+ video_openface_csv=args.openface_csv,
85
+ transcript=args.transcript,
86
+ ))
87
+
88
+ except Exception as e:
89
+ print(f"Inference error: {e}")
90
+ import traceback
91
+ traceback.print_exc()
92
+ sys.exit(1)
93
+
94
+
95
+ if __name__ == "__main__":
96
+ main()
scripts/serve.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import uvicorn
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ ROOT_DIR = Path(__file__).resolve().parents[1]
7
+ if str(ROOT_DIR) not in sys.path:
8
+ sys.path.insert(0, str(ROOT_DIR))
9
+
10
+ from src.api.app import create_app
11
+ from src.utils.config import load_config
12
+
13
+
14
+ def main() -> None:
15
+ parser = argparse.ArgumentParser(description="Serve ModelV2 inference API")
16
+ parser.add_argument("--config", type=Path, default=Path("configs/inference.yaml"))
17
+ parser.add_argument("--host", type=str, default="0.0.0.0")
18
+ parser.add_argument("--port", type=int, default=None)
19
+ parser.add_argument("--log-level", type=str, default="info")
20
+ parser.add_argument("--reload", action="store_true")
21
+ args = parser.parse_args()
22
+ config = load_config(args.config)
23
+ if args.port is not None:
24
+ config["api"]["port"] = args.port
25
+ checkpoint_path = config["artifacts"]["checkpoint_path"]
26
+ model_config = config["model"]
27
+ device = config.get("inference", {}).get("device", "auto")
28
+ app = create_app(str(checkpoint_path), model_config, device=device)
29
+
30
+ # We must pass the import string 'src.api.app:create_app' if we want reload to work,
31
+ # but since app is already created, reload might not work properly without restructuring.
32
+ # We'll just pass the app directly.
33
+ uvicorn.run(app, host=args.host, port=config.get("api", {}).get("port", 8000), log_level=args.log_level.lower())
34
+
35
+
36
+ if __name__ == "__main__":
37
+ main()
scripts/smoke_infer_test.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Smoke test: load inference config and instantiate ModelV2Inferencer using CPU."""
3
+ import sys
4
+ from pathlib import Path
5
+ import traceback
6
+
7
+ from src.utils.config import load_config
8
+
9
+
10
+ def main():
11
+ try:
12
+ repo_root = Path(__file__).resolve().parents[1]
13
+ cfg_path = repo_root / "configs" / "inference.yaml"
14
+ cfg = load_config(cfg_path)
15
+ # Force CPU to avoid CUDA/device autodetection issues
16
+ cfg.setdefault("inference", {})["device"] = "cpu"
17
+
18
+ checkpoint = Path(cfg["artifacts"]["checkpoint_path"]) if "artifacts" in cfg else Path("checkpoints/best_model.pt")
19
+ if not checkpoint.is_absolute():
20
+ checkpoint = (repo_root / checkpoint).resolve()
21
+
22
+ print("Config loaded. Checkpoint:", checkpoint)
23
+ print("Exists:", checkpoint.exists())
24
+
25
+ from src.inference.inferencer import ModelV2Inferencer
26
+
27
+ model_cfg = cfg.get("model", {})
28
+ infer = ModelV2Inferencer(checkpoint, model_cfg, device="cpu")
29
+ print("Model instantiated on device:", infer.device)
30
+ print("Smoke test OK")
31
+ except Exception:
32
+ traceback.print_exc()
33
+ sys.exit(2)
34
+
35
+
36
+ if __name__ == "__main__":
37
+ main()
scripts/train.py ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import logging
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ ROOT_DIR = Path(__file__).resolve().parents[1]
7
+ if str(ROOT_DIR) not in sys.path:
8
+ sys.path.insert(0, str(ROOT_DIR))
9
+
10
+ import yaml
11
+ import torch
12
+ from torch.utils.data import DataLoader
13
+
14
+ from src.dataset.builder import DatasetBuilder
15
+ from src.dataset.collate import multimodal_collate_fn
16
+ from src.models.multimodal_model import MultimodalDepressionModel
17
+ from src.training.trainer import Trainer
18
+ from src.utils.logging import configure_logging
19
+
20
+
21
+ def load_config(path: Path) -> dict:
22
+ with open(path, "r", encoding="utf-8") as handle:
23
+ return yaml.safe_load(handle)
24
+
25
+
26
+ def main() -> None:
27
+ parser = argparse.ArgumentParser(description="Train ModelV2 multimodal depression model")
28
+ parser.add_argument("--config", type=Path, default=Path("configs/training.yaml"))
29
+ args = parser.parse_args()
30
+
31
+ config = load_config(args.config)
32
+ run_dir = configure_logging(Path(config.get("logging", {}).get("log_dir", "logs")), experiment_name=config.get("logging", {}).get("experiment_name", "trimodal"))
33
+ logger = logging.getLogger(__name__)
34
+
35
+ train_dataset = DatasetBuilder.build_from_csv(
36
+ split_csv_path=Path(config["dataset"]["train_split"]),
37
+ feature_dir=Path(config["dataset"]["feature_dir"]),
38
+ id_column=config["dataset"].get("id_column", "participant_id"),
39
+ label_column=config["dataset"].get("label_column", "phq_total"),
40
+ augment=True,
41
+ )
42
+ val_dataset = DatasetBuilder.build_from_csv(
43
+ split_csv_path=Path(config["dataset"]["val_split"]),
44
+ feature_dir=Path(config["dataset"]["feature_dir"]),
45
+ id_column=config["dataset"].get("id_column", "participant_id"),
46
+ label_column=config["dataset"].get("label_column", "phq_total"),
47
+ augment=False,
48
+ )
49
+
50
+ train_loader = DataLoader(
51
+ train_dataset,
52
+ batch_size=int(config["training"].get("batch_size", 8)),
53
+ shuffle=True,
54
+ num_workers=int(config["training"].get("num_workers", 2)),
55
+ collate_fn=multimodal_collate_fn,
56
+ )
57
+ val_loader = DataLoader(
58
+ val_dataset,
59
+ batch_size=int(config["training"].get("batch_size", 8)),
60
+ shuffle=False,
61
+ num_workers=int(config["training"].get("num_workers", 2)),
62
+ collate_fn=multimodal_collate_fn,
63
+ )
64
+
65
+ model_config = config.get("model", {})
66
+ model = MultimodalDepressionModel(**model_config)
67
+ optimizer = torch.optim.AdamW(
68
+ model.parameters(),
69
+ lr=float(config["training"].get("learning_rate", 3e-4)),
70
+ weight_decay=float(config["training"].get("weight_decay", 1e-4)),
71
+ )
72
+ scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
73
+ optimizer,
74
+ mode="min",
75
+ factor=float(config["training"].get("lr_factor", 0.5)),
76
+ patience=int(config["training"].get("lr_patience", 3)),
77
+ )
78
+
79
+ trainer = Trainer(
80
+ model=model,
81
+ train_loader=train_loader,
82
+ val_loader=val_loader,
83
+ optimizer=optimizer,
84
+ scheduler=scheduler,
85
+ config=config["training"],
86
+ )
87
+ logger.info("Starting ModelV2 training")
88
+ history = trainer.train()
89
+ logger.info(f"Training complete: {history}")
90
+
91
+
92
+ if __name__ == "__main__":
93
+ main()
scripts/train_v2.py ADDED
@@ -0,0 +1,378 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Improved training script for multimodal depression detection.
3
+
4
+ Features:
5
+ - Consistent feature pipeline with validation
6
+ - Advanced regularization (EMA, checkpoint averaging, curriculum learning)
7
+ - Comprehensive logging and checkpointing
8
+ - Small dataset optimization
9
+ """
10
+
11
+ import argparse
12
+ import logging
13
+ import sys
14
+ from pathlib import Path
15
+ import numpy as np
16
+
17
+ def convert_numpy(obj):
18
+ if isinstance(obj, np.integer):
19
+ return int(obj)
20
+ if isinstance(obj, np.floating):
21
+ return float(obj)
22
+ if isinstance(obj, np.ndarray):
23
+ return obj.tolist()
24
+ if isinstance(obj, dict):
25
+ return {k: convert_numpy(v) for k, v in obj.items()}
26
+ if isinstance(obj, list):
27
+ return [convert_numpy(i) for i in obj]
28
+ return obj
29
+
30
+ ROOT_DIR = Path(__file__).resolve().parents[1]
31
+ if str(ROOT_DIR) not in sys.path:
32
+ sys.path.insert(0, str(ROOT_DIR))
33
+
34
+ import yaml
35
+ import torch
36
+ from torch.utils.data import DataLoader
37
+
38
+ from src.dataset.builder import DatasetBuilder
39
+ from src.dataset.collate import multimodal_collate_fn
40
+ from src.dataset.multimodal_dataset import MultimodalSample
41
+ from src.models.multimodal_model import MultimodalDepressionModel
42
+ from src.training.trainer import Trainer
43
+ from src.training.optimization import EMA, WarmupScheduler, CurriculumLearning
44
+ from src.features.specs import FeatureSpecification
45
+ from src.features.preprocessing import FeaturePreprocessor
46
+ from src.inference.validation import TrainInferenceConsistencyChecker, FeatureCheckpoint
47
+ from src.utils.logging import configure_logging
48
+
49
+
50
+ def load_config(path: Path) -> dict:
51
+ """Load YAML configuration."""
52
+ with open(path, "r", encoding="utf-8") as f:
53
+ return yaml.safe_load(f)
54
+
55
+
56
+ def setup_feature_specs(config: dict) -> FeatureSpecification:
57
+ """Set up feature specifications from config."""
58
+ # Use defaults or override from config
59
+ feature_config = config.get("features", {})
60
+
61
+ # Create feature specs matching the config
62
+ from src.features.specs import ModalitySpec
63
+
64
+ # Determine which specs to use based on dimensions
65
+ audio_dim = feature_config.get("audio_dim", 39)
66
+ video_dim = feature_config.get("video_dim", 38)
67
+ text_dim = feature_config.get("text_dim", 384)
68
+
69
+ # For now, use standard specs (can be enhanced to support other combinations)
70
+ spec = FeatureSpecification(
71
+ audio_spec=ModalitySpec(name="audio", expected_dim=audio_dim, temporal=True),
72
+ video_spec=ModalitySpec(name="video", expected_dim=video_dim, temporal=True),
73
+ text_spec=ModalitySpec(name="text", expected_dim=text_dim, temporal=True),
74
+ )
75
+
76
+ # Removed strict assertions because PCA determines final dimensions dynamically
77
+ return spec
78
+
79
+
80
+ def setup_preprocessors(config: dict, feature_spec: FeatureSpecification) -> dict:
81
+ """Set up feature preprocessors (normalization, PCA)."""
82
+ preprocessors = {}
83
+
84
+ # Audio preprocessor
85
+ audio_prep = FeaturePreprocessor(
86
+ normalizer_type="standard",
87
+ apply_pca=True,
88
+ pca_components=0.80,
89
+ pca_whiten=True,
90
+ )
91
+ preprocessors["audio"] = audio_prep
92
+
93
+ # Video preprocessor
94
+ video_prep = FeaturePreprocessor(
95
+ normalizer_type="standard",
96
+ apply_pca=True,
97
+ pca_components=0.80,
98
+ pca_whiten=True,
99
+ )
100
+ preprocessors["video"] = video_prep
101
+
102
+ # Text preprocessor (Aggressive reduction)
103
+ text_prep = FeaturePreprocessor(
104
+ normalizer_type="standard",
105
+ apply_pca=True,
106
+ pca_components=0.80,
107
+ pca_whiten=True,
108
+ )
109
+ preprocessors["text"] = text_prep
110
+
111
+ return preprocessors
112
+
113
+
114
+ def _try_load_preprocessors(checkpoint_dir: Path, logger: logging.Logger) -> dict | None:
115
+ """Try to load already-saved PCA preprocessors from the most recent checkpoint.
116
+
117
+ Reuses the PCA transform computed in a previous training run so that:
118
+ - dimensions stay consistent with the saved best_model.pt
119
+ - no refitting noise between runs
120
+ - faster startup
121
+
122
+ Returns None if no valid checkpoint exists.
123
+ """
124
+ # Preference order: multimodal_v2 (best validated run), then any other
125
+ candidates = ["multimodal_v3", "multimodal_v2", "multimodal_v1"]
126
+ for name in candidates:
127
+ prep_dir = checkpoint_dir / name / "preprocessors"
128
+ if not prep_dir.exists():
129
+ continue
130
+ missing = [m for m in ["audio", "video", "text"] if not (prep_dir / f"{m}_preprocessor.pkl").exists()]
131
+ if missing:
132
+ continue
133
+ try:
134
+ from src.inference.validation import FeatureCheckpoint
135
+ manager = FeatureCheckpoint(checkpoint_dir)
136
+ preps = manager.load_preprocessors(name)
137
+ if all(preps.get(m) and preps[m].output_dim for m in ["audio", "video", "text"]):
138
+ logger.info(
139
+ f"Loaded existing PCA preprocessors from '{name}' checkpoint: "
140
+ f"audio={preps['audio'].output_dim}d "
141
+ f"video={preps['video'].output_dim}d "
142
+ f"text={preps['text'].output_dim}d"
143
+ )
144
+ return preps
145
+ except Exception as exc:
146
+ logger.warning(f"Could not load preprocessors from '{name}': {exc}")
147
+ return None
148
+
149
+
150
+ def create_datasets_with_preprocessing(
151
+ config: dict,
152
+ preprocessors: dict,
153
+ feature_spec: FeatureSpecification,
154
+ logger: logging.Logger,
155
+ checkpoint_dir: Path | None = None,
156
+ ) -> tuple:
157
+ """Create training and validation datasets with preprocessing.
158
+
159
+ Tries to load already-saved PCA preprocessors first. Only refits if none exist.
160
+ """
161
+
162
+ reg_cfg = config.get("regularization", {})
163
+ feat_noise = float(reg_cfg.get("feature_noise_std", 0.01))
164
+ temp_drop = float(reg_cfg.get("temporal_dropout_rate", 0.10))
165
+
166
+ train_dataset_raw = DatasetBuilder.build_from_csv(
167
+ split_csv_path=Path(config["dataset"]["train_split"]),
168
+ feature_dir=Path(config["dataset"]["feature_dir"]),
169
+ id_column=config["dataset"].get("id_column", "participant_id"),
170
+ label_column=config["dataset"].get("label_column", "phq_total"),
171
+ augment=True,
172
+ temporal_dropout_rate=temp_drop,
173
+ feature_noise_std=feat_noise,
174
+ )
175
+
176
+ val_dataset_raw = DatasetBuilder.build_from_csv(
177
+ split_csv_path=Path(config["dataset"]["val_split"]),
178
+ feature_dir=Path(config["dataset"]["feature_dir"]),
179
+ id_column=config["dataset"].get("id_column", "participant_id"),
180
+ label_column=config["dataset"].get("label_column", "phq_total"),
181
+ augment=False,
182
+ temporal_dropout_rate=0.0,
183
+ feature_noise_std=0.0,
184
+ )
185
+
186
+ logger.info(f"Loaded {len(train_dataset_raw)} training samples")
187
+ logger.info(f"Loaded {len(val_dataset_raw)} validation samples")
188
+
189
+ # Try to reuse already-saved PCA preprocessors (avoids refitting noise)
190
+ saved = _try_load_preprocessors(checkpoint_dir or Path("checkpoints"), logger)
191
+ if saved:
192
+ preprocessors = saved
193
+ logger.info("Using saved PCA preprocessors — skipping refit.")
194
+ else:
195
+ # Fit PCA preprocessors fresh on training data
196
+ logger.info("No saved preprocessors found. Fitting PCA from scratch...")
197
+ for modality in ["audio", "video", "text"]:
198
+ features_list = [
199
+ s.get(modality) for s in train_dataset_raw.samples
200
+ if s.get(modality) is not None
201
+ ]
202
+ if features_list:
203
+ all_features = np.vstack(features_list)
204
+ preprocessors[modality].fit(all_features)
205
+ logger.info(
206
+ f" {modality}: PCA fitted → {preprocessors[modality].output_dim}d "
207
+ f"(from {all_features.shape[1]}d, {all_features.shape[0]} rows)"
208
+ )
209
+
210
+ # Apply preprocessors to both splits
211
+ logger.info("Applying PCA preprocessors to datasets...")
212
+ for split_name, dataset in [("train", train_dataset_raw), ("val", val_dataset_raw)]:
213
+ for sample in dataset.samples:
214
+ for modality in ["audio", "video", "text"]:
215
+ if sample.get(modality) is not None:
216
+ sample[modality] = preprocessors[modality].transform(sample[modality])
217
+ logger.info(
218
+ f" {split_name}: applied PCA — audio now {dataset.samples[0]['audio'].shape[1] if dataset.samples[0].get('audio') is not None else 'N/A'}d"
219
+ )
220
+
221
+ return train_dataset_raw, val_dataset_raw, preprocessors
222
+
223
+
224
+ def main() -> None:
225
+ """Main training function."""
226
+ parser = argparse.ArgumentParser(description="Train improved multimodal depression model")
227
+ parser.add_argument("--config", type=Path, default=Path("configs/training.yaml"))
228
+ parser.add_argument("--experiment-name", type=str, default="multimodal_v2")
229
+ parser.add_argument("--use-ema", action="store_true", help="Use exponential moving average")
230
+ parser.add_argument("--seed", type=int, default=42, help="Random seed for reproducibility")
231
+ args = parser.parse_args()
232
+
233
+ # Fix random seed for reproducibility
234
+ import random
235
+ random.seed(args.seed)
236
+ np.random.seed(args.seed)
237
+ torch.manual_seed(args.seed)
238
+ if torch.cuda.is_available():
239
+ torch.cuda.manual_seed_all(args.seed)
240
+
241
+ # Load config
242
+ config = load_config(args.config)
243
+
244
+ # Set up logging
245
+ run_dir = configure_logging(
246
+ Path(config.get("logging", {}).get("log_dir", "logs")),
247
+ experiment_name=args.experiment_name,
248
+ )
249
+ logger = logging.getLogger(__name__)
250
+
251
+ logger.info(f"Starting training: {args.experiment_name}")
252
+ logger.info(f"Config: {args.config}")
253
+
254
+ # Feature specification and preprocessing
255
+ logger.info("Setting up feature pipeline...")
256
+ feature_spec = setup_feature_specs(config)
257
+ logger.info(f"Feature spec - Audio: {feature_spec.audio_dim}D, Video: {feature_spec.video_dim}D, Text: {feature_spec.text_dim}D")
258
+
259
+ preprocessors = setup_preprocessors(config, feature_spec)
260
+
261
+ checkpoint_dir = Path(config["training"].get("checkpoint_dir", "checkpoints"))
262
+
263
+ # Create datasets — loads existing PCA preprocessors if available
264
+ train_dataset, val_dataset, preprocessors = create_datasets_with_preprocessing(
265
+ config,
266
+ preprocessors,
267
+ feature_spec,
268
+ logger,
269
+ checkpoint_dir=checkpoint_dir,
270
+ )
271
+
272
+ # Update model config dynamically based on PCA output dims
273
+ if preprocessors["audio"].output_dim:
274
+ config["model"]["audio_dim"] = preprocessors["audio"].output_dim
275
+ if preprocessors["video"].output_dim:
276
+ config["model"]["video_dim"] = preprocessors["video"].output_dim
277
+ if preprocessors["text"].output_dim:
278
+ config["model"]["text_dim"] = preprocessors["text"].output_dim
279
+
280
+ logger.info(f"Final model dims - Audio: {config['model']['audio_dim']}, Video: {config['model']['video_dim']}, Text: {config['model']['text_dim']}")
281
+
282
+ # Create dataloaders
283
+ train_loader = DataLoader(
284
+ train_dataset,
285
+ batch_size=int(config["training"].get("batch_size", 8)),
286
+ shuffle=True,
287
+ num_workers=int(config["training"].get("num_workers", 2)),
288
+ collate_fn=multimodal_collate_fn,
289
+ )
290
+ val_loader = DataLoader(
291
+ val_dataset,
292
+ batch_size=int(config["training"].get("batch_size", 8)),
293
+ shuffle=False,
294
+ num_workers=int(config["training"].get("num_workers", 2)),
295
+ collate_fn=multimodal_collate_fn,
296
+ )
297
+
298
+ # Model
299
+ model_config = config.get("model", {})
300
+ model = MultimodalDepressionModel(**model_config)
301
+ logger.info(f"Model parameters: {sum(p.numel() for p in model.parameters()):,}")
302
+
303
+ # Optimizer and scheduler
304
+ optimizer = torch.optim.AdamW(
305
+ model.parameters(),
306
+ lr=float(config["training"].get("learning_rate", 1e-4)),
307
+ weight_decay=float(config["training"].get("weight_decay", 1e-4)),
308
+ )
309
+
310
+ # Warmup scheduler
311
+ warmup_scheduler = WarmupScheduler(
312
+ optimizer,
313
+ warmup_epochs=5,
314
+ total_epochs=int(config["training"].get("max_epochs", 100)),
315
+ base_lr=float(config["training"].get("learning_rate", 1e-4)),
316
+ )
317
+
318
+ # Reduce on plateau scheduler
319
+ plateau_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
320
+ optimizer,
321
+ mode="min",
322
+ factor=float(config["training"].get("lr_factor", 0.5)),
323
+ patience=int(config["training"].get("lr_patience", 5)),
324
+ )
325
+
326
+ # EMA if enabled
327
+ ema = None
328
+ if args.use_ema:
329
+ ema = EMA(model, decay=0.999)
330
+ logger.info("Using exponential moving average")
331
+
332
+ # Merge loss weights into training config for Trainer
333
+ training_config = config["training"].copy()
334
+ if "loss" in config:
335
+ training_config.update(config["loss"])
336
+
337
+ # Trainer
338
+ trainer = Trainer(
339
+ model=model,
340
+ train_loader=train_loader,
341
+ val_loader=val_loader,
342
+ optimizer=optimizer,
343
+ scheduler=plateau_scheduler,
344
+ config=training_config,
345
+ )
346
+
347
+ # Training
348
+ logger.info("Starting training...")
349
+ history = trainer.train()
350
+
351
+ # Save best model info
352
+ best_loss = getattr(trainer, "best_loss", None)
353
+ best_ccc = getattr(trainer, "best_ccc", None)
354
+ logger.info("Training complete!")
355
+ logger.info(f"Best CCC: {best_ccc:.4f}" if best_ccc else "Best CCC: N/A")
356
+
357
+ # Write result.json
358
+ result_data = {
359
+ "best_ccc": float(best_ccc) if best_ccc else 0.0,
360
+ "best_loss": float(best_loss) if best_loss else 0.0,
361
+ "experiment_name": args.experiment_name,
362
+ "config": convert_numpy(config),
363
+ "run_dir": str(run_dir),
364
+ }
365
+ import json
366
+ with open("result.json", "w") as f:
367
+ json.dump(result_data, f, indent=4)
368
+ logger.info(f"Best loss: {history.get('best_loss', 'N/A')}")
369
+
370
+ # Save preprocessors for inference
371
+ checkpoint_manager = FeatureCheckpoint(Path(config["training"].get("checkpoint_dir", "checkpoints")))
372
+ checkpoint_manager.save_preprocessors(preprocessors, args.experiment_name)
373
+ checkpoint_manager.save_feature_spec(feature_spec, args.experiment_name)
374
+ logger.info(f"Saved preprocessors and feature spec to {args.experiment_name}")
375
+
376
+
377
+ if __name__ == "__main__":
378
+ main()
src/__init__.py ADDED
File without changes
src/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (120 Bytes). View file
 
src/api/__init__.py ADDED
File without changes
src/api/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (124 Bytes). View file
 
src/api/__pycache__/app.cpython-312.pyc ADDED
Binary file (5.05 kB). View file
 
src/api/__pycache__/routes.cpython-312.pyc ADDED
Binary file (14 kB). View file