mindscope-Model / src /models /multimodal_model.py
koushik2109's picture
Model Changes
8d7b4b1
Raw History Blame Contribute Delete
9.71 kB
from __future__ import annotations
import torch
import torch.nn as nn
from typing import Optional
from src.models.encoders import (
AttentionPooling,
BiGRUEncoder,
ConvGRUEncoder,
TemporalConvEncoder,
TransformerEncoder,
)
from src.models.fusion import MultimodalFusion
from src.models.heads import ClassificationHead, ConfidenceHead, QuestionHead, RegressionHead
class MultimodalDepressionModel(nn.Module):
def __init__(
self,
audio_dim: int = 512,
video_dim: int = 512,
text_dim: int = 384,
fusion_dim: int = 256,
num_questions: int = 8,
encoder_type: str = "transformer",
pooling_type: str = "attention",
num_heads: int = 4,
num_layers: int = 2,
dropout: float = 0.2,
fusion_mode: str = "hybrid",
modality_dropout: float = 0.1,
):
super().__init__()
self.encoder_type = encoder_type
self.pooling_type = pooling_type
self.audio_input_dim = audio_dim
self.video_input_dim = video_dim
self.text_input_dim = text_dim
self.audio_encoder = self._build_encoder(audio_dim, fusion_dim, dropout, num_layers, num_heads)
self.video_encoder = self._build_encoder(video_dim, fusion_dim, dropout, num_layers, num_heads)
self.text_encoder = self._build_encoder(text_dim, fusion_dim, dropout, num_layers, num_heads)
self.audio_projection = self._build_projection(audio_dim, fusion_dim)
self.video_projection = self._build_projection(video_dim, fusion_dim)
self.text_projection = self._build_projection(text_dim, fusion_dim)
self.audio_missing = nn.Parameter(torch.zeros(fusion_dim))
self.video_missing = nn.Parameter(torch.zeros(fusion_dim))
self.text_missing = nn.Parameter(torch.zeros(fusion_dim))
# Persistent attention poolers — one per modality so weights ARE trained.
# (previously created fresh each forward pass → random, never trained → CCC gap)
if self.pooling_type == "attention":
self.audio_pooler = AttentionPooling(fusion_dim)
self.video_pooler = AttentionPooling(fusion_dim)
self.text_pooler = AttentionPooling(fusion_dim)
self.fusion = MultimodalFusion(
embed_dim=fusion_dim,
fusion_dim=fusion_dim,
mode=fusion_mode,
dropout=dropout,
)
self.phq_head = RegressionHead(fusion_dim)
self.question_head = QuestionHead(fusion_dim, num_questions)
self.classification_head = ClassificationHead(fusion_dim)
self.confidence_head = ConfidenceHead(fusion_dim)
self.modality_dropout = modality_dropout
def _build_encoder(
self,
input_dim: int,
hidden_dim: int,
dropout: float,
num_layers: int,
num_heads: int,
) -> nn.Module:
if self.encoder_type == "conv":
return TemporalConvEncoder(input_dim, hidden_dim, dropout=dropout)
if self.encoder_type == "conv_gru":
return ConvGRUEncoder(input_dim, hidden_dim, num_layers=num_layers, dropout=dropout)
if self.encoder_type == "bigru":
return BiGRUEncoder(input_dim, hidden_dim, num_layers=num_layers, dropout=dropout)
return TransformerEncoder(input_dim, num_layers=num_layers, num_heads=num_heads, dropout=dropout)
def _build_projection(self, input_dim: int, hidden_dim: int) -> nn.Module:
if self.encoder_type == "bigru":
return nn.Linear(hidden_dim * 2, hidden_dim)
if self.encoder_type == "transformer":
return nn.Linear(input_dim, hidden_dim)
return nn.Linear(hidden_dim, hidden_dim)
@staticmethod
def _mean_pool(x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
mask = mask.unsqueeze(-1).float()
summed = (x * mask).sum(dim=1)
length = mask.sum(dim=1).clamp(min=1.0)
return summed / length
def _pooled_representation(
self, x: torch.Tensor, mask: torch.Tensor, pooler: Optional[nn.Module] = None
) -> torch.Tensor:
if self.pooling_type == "attention" and pooler is not None:
return pooler(x, mask)
return self._mean_pool(x, mask)
def _encode_modality(
self,
x: Optional[torch.Tensor],
mask: Optional[torch.Tensor],
encoder: nn.Module,
projector: nn.Module,
missing_token: nn.Parameter,
pooler: Optional[nn.Module] = None,
) -> tuple[torch.Tensor, torch.Tensor]:
if x is None or x.numel() == 0 or mask is None:
batch = x.shape[0] if x is not None else 1
return missing_token.unsqueeze(0).expand(batch, -1), torch.zeros(batch, dtype=torch.bool, device=missing_token.device)
mask = mask.to(dtype=torch.bool)
present = mask.any(dim=1)
encoded = encoder(x, mask)
pooled = self._pooled_representation(encoded, mask, pooler)
projected = projector(pooled)
if not present.all():
fallback = missing_token.unsqueeze(0).expand(projected.size(0), -1)
projected = projected * present.unsqueeze(-1).to(projected.dtype) + fallback * (~present).unsqueeze(-1).to(projected.dtype)
return projected, present
def _drop_modality(
self,
modality_repr: torch.Tensor,
present: torch.Tensor,
missing_token: nn.Parameter,
) -> tuple[torch.Tensor, torch.Tensor]:
if not self.training or self.modality_dropout <= 0.0:
return modality_repr, present
keep = torch.rand(modality_repr.size(0), device=modality_repr.device) > self.modality_dropout
keep = keep & present
if keep.all():
return modality_repr, present
fallback = missing_token.unsqueeze(0).expand_as(modality_repr)
modality_repr = modality_repr * keep.unsqueeze(-1).to(modality_repr.dtype) + fallback * (~keep).unsqueeze(-1).to(modality_repr.dtype)
return modality_repr, keep
def forward(
self,
audio: Optional[torch.Tensor],
video: Optional[torch.Tensor],
text: Optional[torch.Tensor],
audio_mask: Optional[torch.Tensor],
video_mask: Optional[torch.Tensor],
text_mask: Optional[torch.Tensor],
) -> dict[str, torch.Tensor]:
audio_repr, audio_present = self._encode_modality(
audio,
audio_mask,
self.audio_encoder,
self.audio_projection,
self.audio_missing,
pooler=getattr(self, "audio_pooler", None),
)
video_repr, video_present = self._encode_modality(
video,
video_mask,
self.video_encoder,
self.video_projection,
self.video_missing,
pooler=getattr(self, "video_pooler", None),
)
text_repr, text_present = self._encode_modality(
text,
text_mask,
self.text_encoder,
self.text_projection,
self.text_missing,
pooler=getattr(self, "text_pooler", None),
)
if self.training and self.modality_dropout > 0.0:
a_keep = torch.rand(audio_repr.size(0), device=audio_repr.device) > self.modality_dropout
v_keep = torch.rand(video_repr.size(0), device=video_repr.device) > self.modality_dropout
t_keep = torch.rand(text_repr.size(0), device=text_repr.device) > self.modality_dropout
# If all are false, force one to be true among the ones that are actually present
a_keep = a_keep & audio_present
v_keep = v_keep & video_present
t_keep = t_keep & text_present
all_dropped = ~(a_keep | v_keep | t_keep)
# For rows where all dropped, try text, then audio, then video
if all_dropped.any():
t_keep = t_keep | (all_dropped & text_present)
all_dropped_still = all_dropped & ~text_present
a_keep = a_keep | (all_dropped_still & audio_present)
all_dropped_still = all_dropped_still & ~audio_present
v_keep = v_keep | (all_dropped_still & video_present)
def apply_drop(repr_t, present_t, keep_t, missing_t):
if keep_t.all(): return repr_t, present_t
fallback = missing_t.unsqueeze(0).expand_as(repr_t)
repr_out = repr_t * keep_t.unsqueeze(-1).to(repr_t.dtype) + fallback * (~keep_t).unsqueeze(-1).to(repr_t.dtype)
return repr_out, keep_t
audio_repr, audio_present = apply_drop(audio_repr, audio_present, a_keep, self.audio_missing)
video_repr, video_present = apply_drop(video_repr, video_present, v_keep, self.video_missing)
text_repr, text_present = apply_drop(text_repr, text_present, t_keep, self.text_missing)
modality_present = torch.stack([audio_present, video_present, text_present], dim=1)
fused, modality_scores, entropy, confidence = self.fusion(
audio_repr,
video_repr,
text_repr,
modality_present=modality_present,
)
return {
"phq_total": self.phq_head(fused).squeeze(-1),
"phq_questions": self.question_head(fused),
"classification": self.classification_head(fused).squeeze(-1),
"confidence": self.confidence_head(fused).squeeze(-1),
"modality_scores": modality_scores,
"modality_confidence": confidence,
"fusion_entropy": entropy,
"entropy": entropy,
}