mindscope-Model / scripts /train_v2.py
koushik2109's picture
Model Changes
8d7b4b1
Raw History Blame Contribute Delete
14.1 kB
#!/usr/bin/env python3
"""Improved training script for multimodal depression detection.
Features:
- Consistent feature pipeline with validation
- Advanced regularization (EMA, checkpoint averaging, curriculum learning)
- Comprehensive logging and checkpointing
- Small dataset optimization
"""
import argparse
import logging
import sys
from pathlib import Path
import numpy as np
def convert_numpy(obj):
if isinstance(obj, np.integer):
return int(obj)
if isinstance(obj, np.floating):
return float(obj)
if isinstance(obj, np.ndarray):
return obj.tolist()
if isinstance(obj, dict):
return {k: convert_numpy(v) for k, v in obj.items()}
if isinstance(obj, list):
return [convert_numpy(i) for i in obj]
return obj
ROOT_DIR = Path(__file__).resolve().parents[1]
if str(ROOT_DIR) not in sys.path:
sys.path.insert(0, str(ROOT_DIR))
import yaml
import torch
from torch.utils.data import DataLoader
from src.dataset.builder import DatasetBuilder
from src.dataset.collate import multimodal_collate_fn
from src.dataset.multimodal_dataset import MultimodalSample
from src.models.multimodal_model import MultimodalDepressionModel
from src.training.trainer import Trainer
from src.training.optimization import EMA, WarmupScheduler, CurriculumLearning
from src.features.specs import FeatureSpecification
from src.features.preprocessing import FeaturePreprocessor
from src.inference.validation import TrainInferenceConsistencyChecker, FeatureCheckpoint
from src.utils.logging import configure_logging
def load_config(path: Path) -> dict:
"""Load YAML configuration."""
with open(path, "r", encoding="utf-8") as f:
return yaml.safe_load(f)
def setup_feature_specs(config: dict) -> FeatureSpecification:
"""Set up feature specifications from config."""
# Use defaults or override from config
feature_config = config.get("features", {})
# Create feature specs matching the config
from src.features.specs import ModalitySpec
# Determine which specs to use based on dimensions
audio_dim = feature_config.get("audio_dim", 39)
video_dim = feature_config.get("video_dim", 38)
text_dim = feature_config.get("text_dim", 384)
# For now, use standard specs (can be enhanced to support other combinations)
spec = FeatureSpecification(
audio_spec=ModalitySpec(name="audio", expected_dim=audio_dim, temporal=True),
video_spec=ModalitySpec(name="video", expected_dim=video_dim, temporal=True),
text_spec=ModalitySpec(name="text", expected_dim=text_dim, temporal=True),
)
# Removed strict assertions because PCA determines final dimensions dynamically
return spec
def setup_preprocessors(config: dict, feature_spec: FeatureSpecification) -> dict:
"""Set up feature preprocessors (normalization, PCA)."""
preprocessors = {}
# Audio preprocessor
audio_prep = FeaturePreprocessor(
normalizer_type="standard",
apply_pca=True,
pca_components=0.80,
pca_whiten=True,
)
preprocessors["audio"] = audio_prep
# Video preprocessor
video_prep = FeaturePreprocessor(
normalizer_type="standard",
apply_pca=True,
pca_components=0.80,
pca_whiten=True,
)
preprocessors["video"] = video_prep
# Text preprocessor (Aggressive reduction)
text_prep = FeaturePreprocessor(
normalizer_type="standard",
apply_pca=True,
pca_components=0.80,
pca_whiten=True,
)
preprocessors["text"] = text_prep
return preprocessors
def _try_load_preprocessors(checkpoint_dir: Path, logger: logging.Logger) -> dict | None:
"""Try to load already-saved PCA preprocessors from the most recent checkpoint.
Reuses the PCA transform computed in a previous training run so that:
- dimensions stay consistent with the saved best_model.pt
- no refitting noise between runs
- faster startup
Returns None if no valid checkpoint exists.
"""
# Preference order: multimodal_v2 (best validated run), then any other
candidates = ["multimodal_v3", "multimodal_v2", "multimodal_v1"]
for name in candidates:
prep_dir = checkpoint_dir / name / "preprocessors"
if not prep_dir.exists():
continue
missing = [m for m in ["audio", "video", "text"] if not (prep_dir / f"{m}_preprocessor.pkl").exists()]
if missing:
continue
try:
from src.inference.validation import FeatureCheckpoint
manager = FeatureCheckpoint(checkpoint_dir)
preps = manager.load_preprocessors(name)
if all(preps.get(m) and preps[m].output_dim for m in ["audio", "video", "text"]):
logger.info(
f"Loaded existing PCA preprocessors from '{name}' checkpoint: "
f"audio={preps['audio'].output_dim}d "
f"video={preps['video'].output_dim}d "
f"text={preps['text'].output_dim}d"
)
return preps
except Exception as exc:
logger.warning(f"Could not load preprocessors from '{name}': {exc}")
return None
def create_datasets_with_preprocessing(
config: dict,
preprocessors: dict,
feature_spec: FeatureSpecification,
logger: logging.Logger,
checkpoint_dir: Path | None = None,
) -> tuple:
"""Create training and validation datasets with preprocessing.
Tries to load already-saved PCA preprocessors first. Only refits if none exist.
"""
reg_cfg = config.get("regularization", {})
feat_noise = float(reg_cfg.get("feature_noise_std", 0.01))
temp_drop = float(reg_cfg.get("temporal_dropout_rate", 0.10))
train_dataset_raw = DatasetBuilder.build_from_csv(
split_csv_path=Path(config["dataset"]["train_split"]),
feature_dir=Path(config["dataset"]["feature_dir"]),
id_column=config["dataset"].get("id_column", "participant_id"),
label_column=config["dataset"].get("label_column", "phq_total"),
augment=True,
temporal_dropout_rate=temp_drop,
feature_noise_std=feat_noise,
)
val_dataset_raw = DatasetBuilder.build_from_csv(
split_csv_path=Path(config["dataset"]["val_split"]),
feature_dir=Path(config["dataset"]["feature_dir"]),
id_column=config["dataset"].get("id_column", "participant_id"),
label_column=config["dataset"].get("label_column", "phq_total"),
augment=False,
temporal_dropout_rate=0.0,
feature_noise_std=0.0,
)
logger.info(f"Loaded {len(train_dataset_raw)} training samples")
logger.info(f"Loaded {len(val_dataset_raw)} validation samples")
# Try to reuse already-saved PCA preprocessors (avoids refitting noise)
saved = _try_load_preprocessors(checkpoint_dir or Path("checkpoints"), logger)
if saved:
preprocessors = saved
logger.info("Using saved PCA preprocessors — skipping refit.")
else:
# Fit PCA preprocessors fresh on training data
logger.info("No saved preprocessors found. Fitting PCA from scratch...")
for modality in ["audio", "video", "text"]:
features_list = [
s.get(modality) for s in train_dataset_raw.samples
if s.get(modality) is not None
]
if features_list:
all_features = np.vstack(features_list)
preprocessors[modality].fit(all_features)
logger.info(
f" {modality}: PCA fitted → {preprocessors[modality].output_dim}d "
f"(from {all_features.shape[1]}d, {all_features.shape[0]} rows)"
)
# Apply preprocessors to both splits
logger.info("Applying PCA preprocessors to datasets...")
for split_name, dataset in [("train", train_dataset_raw), ("val", val_dataset_raw)]:
for sample in dataset.samples:
for modality in ["audio", "video", "text"]:
if sample.get(modality) is not None:
sample[modality] = preprocessors[modality].transform(sample[modality])
logger.info(
f" {split_name}: applied PCA — audio now {dataset.samples[0]['audio'].shape[1] if dataset.samples[0].get('audio') is not None else 'N/A'}d"
)
return train_dataset_raw, val_dataset_raw, preprocessors
def main() -> None:
"""Main training function."""
parser = argparse.ArgumentParser(description="Train improved multimodal depression model")
parser.add_argument("--config", type=Path, default=Path("configs/training.yaml"))
parser.add_argument("--experiment-name", type=str, default="multimodal_v2")
parser.add_argument("--use-ema", action="store_true", help="Use exponential moving average")
parser.add_argument("--seed", type=int, default=42, help="Random seed for reproducibility")
args = parser.parse_args()
# Fix random seed for reproducibility
import random
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
# Load config
config = load_config(args.config)
# Set up logging
run_dir = configure_logging(
Path(config.get("logging", {}).get("log_dir", "logs")),
experiment_name=args.experiment_name,
)
logger = logging.getLogger(__name__)
logger.info(f"Starting training: {args.experiment_name}")
logger.info(f"Config: {args.config}")
# Feature specification and preprocessing
logger.info("Setting up feature pipeline...")
feature_spec = setup_feature_specs(config)
logger.info(f"Feature spec - Audio: {feature_spec.audio_dim}D, Video: {feature_spec.video_dim}D, Text: {feature_spec.text_dim}D")
preprocessors = setup_preprocessors(config, feature_spec)
checkpoint_dir = Path(config["training"].get("checkpoint_dir", "checkpoints"))
# Create datasets — loads existing PCA preprocessors if available
train_dataset, val_dataset, preprocessors = create_datasets_with_preprocessing(
config,
preprocessors,
feature_spec,
logger,
checkpoint_dir=checkpoint_dir,
)
# Update model config dynamically based on PCA output dims
if preprocessors["audio"].output_dim:
config["model"]["audio_dim"] = preprocessors["audio"].output_dim
if preprocessors["video"].output_dim:
config["model"]["video_dim"] = preprocessors["video"].output_dim
if preprocessors["text"].output_dim:
config["model"]["text_dim"] = preprocessors["text"].output_dim
logger.info(f"Final model dims - Audio: {config['model']['audio_dim']}, Video: {config['model']['video_dim']}, Text: {config['model']['text_dim']}")
# Create dataloaders
train_loader = DataLoader(
train_dataset,
batch_size=int(config["training"].get("batch_size", 8)),
shuffle=True,
num_workers=int(config["training"].get("num_workers", 2)),
collate_fn=multimodal_collate_fn,
)
val_loader = DataLoader(
val_dataset,
batch_size=int(config["training"].get("batch_size", 8)),
shuffle=False,
num_workers=int(config["training"].get("num_workers", 2)),
collate_fn=multimodal_collate_fn,
)
# Model
model_config = config.get("model", {})
model = MultimodalDepressionModel(**model_config)
logger.info(f"Model parameters: {sum(p.numel() for p in model.parameters()):,}")
# Optimizer and scheduler
optimizer = torch.optim.AdamW(
model.parameters(),
lr=float(config["training"].get("learning_rate", 1e-4)),
weight_decay=float(config["training"].get("weight_decay", 1e-4)),
)
# Warmup scheduler
warmup_scheduler = WarmupScheduler(
optimizer,
warmup_epochs=5,
total_epochs=int(config["training"].get("max_epochs", 100)),
base_lr=float(config["training"].get("learning_rate", 1e-4)),
)
# Reduce on plateau scheduler
plateau_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
mode="min",
factor=float(config["training"].get("lr_factor", 0.5)),
patience=int(config["training"].get("lr_patience", 5)),
)
# EMA if enabled
ema = None
if args.use_ema:
ema = EMA(model, decay=0.999)
logger.info("Using exponential moving average")
# Merge loss weights into training config for Trainer
training_config = config["training"].copy()
if "loss" in config:
training_config.update(config["loss"])
# Trainer
trainer = Trainer(
model=model,
train_loader=train_loader,
val_loader=val_loader,
optimizer=optimizer,
scheduler=plateau_scheduler,
config=training_config,
)
# Training
logger.info("Starting training...")
history = trainer.train()
# Save best model info
best_loss = getattr(trainer, "best_loss", None)
best_ccc = getattr(trainer, "best_ccc", None)
logger.info("Training complete!")
logger.info(f"Best CCC: {best_ccc:.4f}" if best_ccc else "Best CCC: N/A")
# Write result.json
result_data = {
"best_ccc": float(best_ccc) if best_ccc else 0.0,
"best_loss": float(best_loss) if best_loss else 0.0,
"experiment_name": args.experiment_name,
"config": convert_numpy(config),
"run_dir": str(run_dir),
}
import json
with open("result.json", "w") as f:
json.dump(result_data, f, indent=4)
logger.info(f"Best loss: {history.get('best_loss', 'N/A')}")
# Save preprocessors for inference
checkpoint_manager = FeatureCheckpoint(Path(config["training"].get("checkpoint_dir", "checkpoints")))
checkpoint_manager.save_preprocessors(preprocessors, args.experiment_name)
checkpoint_manager.save_feature_spec(feature_spec, args.experiment_name)
logger.info(f"Saved preprocessors and feature spec to {args.experiment_name}")
if __name__ == "__main__":
main()