Spaces:
Sleeping
Sleeping
Download scripts/train_v2.py from koushik2109/mindscope-Model: direct link, hf CLI and curl.
- Browser
- Download file 14.1 kB
-
https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/scripts/train_v2.py
- Command line
-
hf download hf://spaces/koushik2109/mindscope-Model/scripts/train_v2.py
-
curl -L -o train_v2.py https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/scripts/train_v2.py
14.1 kB
| #!/usr/bin/env python3 | |
| """Improved training script for multimodal depression detection. | |
| Features: | |
| - Consistent feature pipeline with validation | |
| - Advanced regularization (EMA, checkpoint averaging, curriculum learning) | |
| - Comprehensive logging and checkpointing | |
| - Small dataset optimization | |
| """ | |
| import argparse | |
| import logging | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| def convert_numpy(obj): | |
| if isinstance(obj, np.integer): | |
| return int(obj) | |
| if isinstance(obj, np.floating): | |
| return float(obj) | |
| if isinstance(obj, np.ndarray): | |
| return obj.tolist() | |
| if isinstance(obj, dict): | |
| return {k: convert_numpy(v) for k, v in obj.items()} | |
| if isinstance(obj, list): | |
| return [convert_numpy(i) for i in obj] | |
| return obj | |
| ROOT_DIR = Path(__file__).resolve().parents[1] | |
| if str(ROOT_DIR) not in sys.path: | |
| sys.path.insert(0, str(ROOT_DIR)) | |
| import yaml | |
| import torch | |
| from torch.utils.data import DataLoader | |
| from src.dataset.builder import DatasetBuilder | |
| from src.dataset.collate import multimodal_collate_fn | |
| from src.dataset.multimodal_dataset import MultimodalSample | |
| from src.models.multimodal_model import MultimodalDepressionModel | |
| from src.training.trainer import Trainer | |
| from src.training.optimization import EMA, WarmupScheduler, CurriculumLearning | |
| from src.features.specs import FeatureSpecification | |
| from src.features.preprocessing import FeaturePreprocessor | |
| from src.inference.validation import TrainInferenceConsistencyChecker, FeatureCheckpoint | |
| from src.utils.logging import configure_logging | |
| def load_config(path: Path) -> dict: | |
| """Load YAML configuration.""" | |
| with open(path, "r", encoding="utf-8") as f: | |
| return yaml.safe_load(f) | |
| def setup_feature_specs(config: dict) -> FeatureSpecification: | |
| """Set up feature specifications from config.""" | |
| # Use defaults or override from config | |
| feature_config = config.get("features", {}) | |
| # Create feature specs matching the config | |
| from src.features.specs import ModalitySpec | |
| # Determine which specs to use based on dimensions | |
| audio_dim = feature_config.get("audio_dim", 39) | |
| video_dim = feature_config.get("video_dim", 38) | |
| text_dim = feature_config.get("text_dim", 384) | |
| # For now, use standard specs (can be enhanced to support other combinations) | |
| spec = FeatureSpecification( | |
| audio_spec=ModalitySpec(name="audio", expected_dim=audio_dim, temporal=True), | |
| video_spec=ModalitySpec(name="video", expected_dim=video_dim, temporal=True), | |
| text_spec=ModalitySpec(name="text", expected_dim=text_dim, temporal=True), | |
| ) | |
| # Removed strict assertions because PCA determines final dimensions dynamically | |
| return spec | |
| def setup_preprocessors(config: dict, feature_spec: FeatureSpecification) -> dict: | |
| """Set up feature preprocessors (normalization, PCA).""" | |
| preprocessors = {} | |
| # Audio preprocessor | |
| audio_prep = FeaturePreprocessor( | |
| normalizer_type="standard", | |
| apply_pca=True, | |
| pca_components=0.80, | |
| pca_whiten=True, | |
| ) | |
| preprocessors["audio"] = audio_prep | |
| # Video preprocessor | |
| video_prep = FeaturePreprocessor( | |
| normalizer_type="standard", | |
| apply_pca=True, | |
| pca_components=0.80, | |
| pca_whiten=True, | |
| ) | |
| preprocessors["video"] = video_prep | |
| # Text preprocessor (Aggressive reduction) | |
| text_prep = FeaturePreprocessor( | |
| normalizer_type="standard", | |
| apply_pca=True, | |
| pca_components=0.80, | |
| pca_whiten=True, | |
| ) | |
| preprocessors["text"] = text_prep | |
| return preprocessors | |
| def _try_load_preprocessors(checkpoint_dir: Path, logger: logging.Logger) -> dict | None: | |
| """Try to load already-saved PCA preprocessors from the most recent checkpoint. | |
| Reuses the PCA transform computed in a previous training run so that: | |
| - dimensions stay consistent with the saved best_model.pt | |
| - no refitting noise between runs | |
| - faster startup | |
| Returns None if no valid checkpoint exists. | |
| """ | |
| # Preference order: multimodal_v2 (best validated run), then any other | |
| candidates = ["multimodal_v3", "multimodal_v2", "multimodal_v1"] | |
| for name in candidates: | |
| prep_dir = checkpoint_dir / name / "preprocessors" | |
| if not prep_dir.exists(): | |
| continue | |
| missing = [m for m in ["audio", "video", "text"] if not (prep_dir / f"{m}_preprocessor.pkl").exists()] | |
| if missing: | |
| continue | |
| try: | |
| from src.inference.validation import FeatureCheckpoint | |
| manager = FeatureCheckpoint(checkpoint_dir) | |
| preps = manager.load_preprocessors(name) | |
| if all(preps.get(m) and preps[m].output_dim for m in ["audio", "video", "text"]): | |
| logger.info( | |
| f"Loaded existing PCA preprocessors from '{name}' checkpoint: " | |
| f"audio={preps['audio'].output_dim}d " | |
| f"video={preps['video'].output_dim}d " | |
| f"text={preps['text'].output_dim}d" | |
| ) | |
| return preps | |
| except Exception as exc: | |
| logger.warning(f"Could not load preprocessors from '{name}': {exc}") | |
| return None | |
| def create_datasets_with_preprocessing( | |
| config: dict, | |
| preprocessors: dict, | |
| feature_spec: FeatureSpecification, | |
| logger: logging.Logger, | |
| checkpoint_dir: Path | None = None, | |
| ) -> tuple: | |
| """Create training and validation datasets with preprocessing. | |
| Tries to load already-saved PCA preprocessors first. Only refits if none exist. | |
| """ | |
| reg_cfg = config.get("regularization", {}) | |
| feat_noise = float(reg_cfg.get("feature_noise_std", 0.01)) | |
| temp_drop = float(reg_cfg.get("temporal_dropout_rate", 0.10)) | |
| train_dataset_raw = DatasetBuilder.build_from_csv( | |
| split_csv_path=Path(config["dataset"]["train_split"]), | |
| feature_dir=Path(config["dataset"]["feature_dir"]), | |
| id_column=config["dataset"].get("id_column", "participant_id"), | |
| label_column=config["dataset"].get("label_column", "phq_total"), | |
| augment=True, | |
| temporal_dropout_rate=temp_drop, | |
| feature_noise_std=feat_noise, | |
| ) | |
| val_dataset_raw = DatasetBuilder.build_from_csv( | |
| split_csv_path=Path(config["dataset"]["val_split"]), | |
| feature_dir=Path(config["dataset"]["feature_dir"]), | |
| id_column=config["dataset"].get("id_column", "participant_id"), | |
| label_column=config["dataset"].get("label_column", "phq_total"), | |
| augment=False, | |
| temporal_dropout_rate=0.0, | |
| feature_noise_std=0.0, | |
| ) | |
| logger.info(f"Loaded {len(train_dataset_raw)} training samples") | |
| logger.info(f"Loaded {len(val_dataset_raw)} validation samples") | |
| # Try to reuse already-saved PCA preprocessors (avoids refitting noise) | |
| saved = _try_load_preprocessors(checkpoint_dir or Path("checkpoints"), logger) | |
| if saved: | |
| preprocessors = saved | |
| logger.info("Using saved PCA preprocessors — skipping refit.") | |
| else: | |
| # Fit PCA preprocessors fresh on training data | |
| logger.info("No saved preprocessors found. Fitting PCA from scratch...") | |
| for modality in ["audio", "video", "text"]: | |
| features_list = [ | |
| s.get(modality) for s in train_dataset_raw.samples | |
| if s.get(modality) is not None | |
| ] | |
| if features_list: | |
| all_features = np.vstack(features_list) | |
| preprocessors[modality].fit(all_features) | |
| logger.info( | |
| f" {modality}: PCA fitted → {preprocessors[modality].output_dim}d " | |
| f"(from {all_features.shape[1]}d, {all_features.shape[0]} rows)" | |
| ) | |
| # Apply preprocessors to both splits | |
| logger.info("Applying PCA preprocessors to datasets...") | |
| for split_name, dataset in [("train", train_dataset_raw), ("val", val_dataset_raw)]: | |
| for sample in dataset.samples: | |
| for modality in ["audio", "video", "text"]: | |
| if sample.get(modality) is not None: | |
| sample[modality] = preprocessors[modality].transform(sample[modality]) | |
| logger.info( | |
| f" {split_name}: applied PCA — audio now {dataset.samples[0]['audio'].shape[1] if dataset.samples[0].get('audio') is not None else 'N/A'}d" | |
| ) | |
| return train_dataset_raw, val_dataset_raw, preprocessors | |
| def main() -> None: | |
| """Main training function.""" | |
| parser = argparse.ArgumentParser(description="Train improved multimodal depression model") | |
| parser.add_argument("--config", type=Path, default=Path("configs/training.yaml")) | |
| parser.add_argument("--experiment-name", type=str, default="multimodal_v2") | |
| parser.add_argument("--use-ema", action="store_true", help="Use exponential moving average") | |
| parser.add_argument("--seed", type=int, default=42, help="Random seed for reproducibility") | |
| args = parser.parse_args() | |
| # Fix random seed for reproducibility | |
| import random | |
| random.seed(args.seed) | |
| np.random.seed(args.seed) | |
| torch.manual_seed(args.seed) | |
| if torch.cuda.is_available(): | |
| torch.cuda.manual_seed_all(args.seed) | |
| # Load config | |
| config = load_config(args.config) | |
| # Set up logging | |
| run_dir = configure_logging( | |
| Path(config.get("logging", {}).get("log_dir", "logs")), | |
| experiment_name=args.experiment_name, | |
| ) | |
| logger = logging.getLogger(__name__) | |
| logger.info(f"Starting training: {args.experiment_name}") | |
| logger.info(f"Config: {args.config}") | |
| # Feature specification and preprocessing | |
| logger.info("Setting up feature pipeline...") | |
| feature_spec = setup_feature_specs(config) | |
| logger.info(f"Feature spec - Audio: {feature_spec.audio_dim}D, Video: {feature_spec.video_dim}D, Text: {feature_spec.text_dim}D") | |
| preprocessors = setup_preprocessors(config, feature_spec) | |
| checkpoint_dir = Path(config["training"].get("checkpoint_dir", "checkpoints")) | |
| # Create datasets — loads existing PCA preprocessors if available | |
| train_dataset, val_dataset, preprocessors = create_datasets_with_preprocessing( | |
| config, | |
| preprocessors, | |
| feature_spec, | |
| logger, | |
| checkpoint_dir=checkpoint_dir, | |
| ) | |
| # Update model config dynamically based on PCA output dims | |
| if preprocessors["audio"].output_dim: | |
| config["model"]["audio_dim"] = preprocessors["audio"].output_dim | |
| if preprocessors["video"].output_dim: | |
| config["model"]["video_dim"] = preprocessors["video"].output_dim | |
| if preprocessors["text"].output_dim: | |
| config["model"]["text_dim"] = preprocessors["text"].output_dim | |
| logger.info(f"Final model dims - Audio: {config['model']['audio_dim']}, Video: {config['model']['video_dim']}, Text: {config['model']['text_dim']}") | |
| # Create dataloaders | |
| train_loader = DataLoader( | |
| train_dataset, | |
| batch_size=int(config["training"].get("batch_size", 8)), | |
| shuffle=True, | |
| num_workers=int(config["training"].get("num_workers", 2)), | |
| collate_fn=multimodal_collate_fn, | |
| ) | |
| val_loader = DataLoader( | |
| val_dataset, | |
| batch_size=int(config["training"].get("batch_size", 8)), | |
| shuffle=False, | |
| num_workers=int(config["training"].get("num_workers", 2)), | |
| collate_fn=multimodal_collate_fn, | |
| ) | |
| # Model | |
| model_config = config.get("model", {}) | |
| model = MultimodalDepressionModel(**model_config) | |
| logger.info(f"Model parameters: {sum(p.numel() for p in model.parameters()):,}") | |
| # Optimizer and scheduler | |
| optimizer = torch.optim.AdamW( | |
| model.parameters(), | |
| lr=float(config["training"].get("learning_rate", 1e-4)), | |
| weight_decay=float(config["training"].get("weight_decay", 1e-4)), | |
| ) | |
| # Warmup scheduler | |
| warmup_scheduler = WarmupScheduler( | |
| optimizer, | |
| warmup_epochs=5, | |
| total_epochs=int(config["training"].get("max_epochs", 100)), | |
| base_lr=float(config["training"].get("learning_rate", 1e-4)), | |
| ) | |
| # Reduce on plateau scheduler | |
| plateau_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( | |
| optimizer, | |
| mode="min", | |
| factor=float(config["training"].get("lr_factor", 0.5)), | |
| patience=int(config["training"].get("lr_patience", 5)), | |
| ) | |
| # EMA if enabled | |
| ema = None | |
| if args.use_ema: | |
| ema = EMA(model, decay=0.999) | |
| logger.info("Using exponential moving average") | |
| # Merge loss weights into training config for Trainer | |
| training_config = config["training"].copy() | |
| if "loss" in config: | |
| training_config.update(config["loss"]) | |
| # Trainer | |
| trainer = Trainer( | |
| model=model, | |
| train_loader=train_loader, | |
| val_loader=val_loader, | |
| optimizer=optimizer, | |
| scheduler=plateau_scheduler, | |
| config=training_config, | |
| ) | |
| # Training | |
| logger.info("Starting training...") | |
| history = trainer.train() | |
| # Save best model info | |
| best_loss = getattr(trainer, "best_loss", None) | |
| best_ccc = getattr(trainer, "best_ccc", None) | |
| logger.info("Training complete!") | |
| logger.info(f"Best CCC: {best_ccc:.4f}" if best_ccc else "Best CCC: N/A") | |
| # Write result.json | |
| result_data = { | |
| "best_ccc": float(best_ccc) if best_ccc else 0.0, | |
| "best_loss": float(best_loss) if best_loss else 0.0, | |
| "experiment_name": args.experiment_name, | |
| "config": convert_numpy(config), | |
| "run_dir": str(run_dir), | |
| } | |
| import json | |
| with open("result.json", "w") as f: | |
| json.dump(result_data, f, indent=4) | |
| logger.info(f"Best loss: {history.get('best_loss', 'N/A')}") | |
| # Save preprocessors for inference | |
| checkpoint_manager = FeatureCheckpoint(Path(config["training"].get("checkpoint_dir", "checkpoints"))) | |
| checkpoint_manager.save_preprocessors(preprocessors, args.experiment_name) | |
| checkpoint_manager.save_feature_spec(feature_spec, args.experiment_name) | |
| logger.info(f"Saved preprocessors and feature spec to {args.experiment_name}") | |
| if __name__ == "__main__": | |
| main() | |