Spaces:
Sleeping
Sleeping
Download src/models/fusion.py from koushik2109/mindscope-Model: direct link, hf CLI and curl.
- Browser
- Download file 13.6 kB
-
https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/models/fusion.py
- Command line
-
hf download hf://spaces/koushik2109/mindscope-Model/src/models/fusion.py
-
curl -L -o fusion.py https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/models/fusion.py
13.6 kB
| from __future__ import annotations | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| class CrossModalAttention(nn.Module): | |
| """Cross-modal attention mechanism for multimodal fusion. | |
| Allows each modality to attend to other modalities for better integration. | |
| """ | |
| def __init__(self, embed_dim: int, num_heads: int = 4, dropout: float = 0.1): | |
| super().__init__() | |
| self.attention = nn.MultiheadAttention( | |
| embed_dim=embed_dim, | |
| num_heads=num_heads, | |
| dropout=dropout, | |
| batch_first=True, | |
| ) | |
| self.proj = nn.Sequential( | |
| nn.Linear(embed_dim, embed_dim), | |
| nn.GELU(), | |
| nn.Dropout(dropout), | |
| nn.LayerNorm(embed_dim), | |
| ) | |
| def forward( | |
| self, | |
| modality_tokens: torch.Tensor, | |
| key_padding_mask: torch.Tensor | None = None, | |
| ) -> torch.Tensor: | |
| """Apply cross-modal attention. | |
| Args: | |
| modality_tokens: [batch_size, num_modalities, embed_dim] | |
| key_padding_mask: [batch_size, num_modalities] boolean mask | |
| Returns: | |
| Attended tokens [batch_size, num_modalities, embed_dim] | |
| """ | |
| attended, _ = self.attention( | |
| modality_tokens, | |
| modality_tokens, | |
| modality_tokens, | |
| key_padding_mask=key_padding_mask, | |
| ) | |
| return self.proj(attended + modality_tokens) | |
| class ModalityGating(nn.Module): | |
| """Learnable modality gating without fixed floor constraints. | |
| Learns to weight each modality dynamically while respecting presence masks. | |
| """ | |
| def __init__( | |
| self, | |
| feature_dim: int, | |
| hidden_dim: int = 256, | |
| dropout: float = 0.1, | |
| entropy_weight: float = 0.1, | |
| ): | |
| super().__init__() | |
| self.entropy_weight = entropy_weight | |
| # Per-modality gating networks | |
| self.modality_heads = nn.ModuleList( | |
| [ | |
| nn.Sequential( | |
| nn.Linear(feature_dim, hidden_dim), | |
| nn.GELU(), | |
| nn.Dropout(dropout), | |
| nn.Linear(hidden_dim, 1), | |
| ) | |
| for _ in range(3) | |
| ] | |
| ) | |
| def forward( | |
| self, | |
| modality_reprs: list[torch.Tensor], | |
| modality_present: torch.Tensor | None = None, | |
| ) -> tuple[torch.Tensor, torch.Tensor]: | |
| """Compute gating weights for modalities. | |
| Args: | |
| modality_reprs: List of [batch_size, embed_dim] tensors for each modality | |
| modality_present: [batch_size, 3] boolean mask of which modalities are present | |
| Returns: | |
| Tuple of: | |
| - weights: [batch_size, 3] normalized weights | |
| - entropy: scalar entropy regularization loss | |
| """ | |
| # Compute logits for each modality | |
| logits = torch.cat( | |
| [head(mod) for head, mod in zip(self.modality_heads, modality_reprs)], | |
| dim=-1, # [batch_size, 3] | |
| ) | |
| # Apply presence masking before softmax | |
| if modality_present is not None: | |
| # Mask out absent modalities | |
| logits = logits.masked_fill(~modality_present, float("-1e9")) | |
| # Compute weights via softmax | |
| weights = F.softmax(logits, dim=-1) # [batch_size, 3] | |
| # Soft gate floor: ensure every *present* modality gets at least min_w weight. | |
| # This prevents modality collapse (e.g. audio gate → 0.99) during training. | |
| # Text is encouraged toward higher values via the text_gate_weight loss term | |
| # in the loss function — not through a hard architectural floor, which would | |
| # limit the residual learning budget and reduce overall CCC. | |
| min_w = 0.15 | |
| if modality_present is not None: | |
| present_f = modality_present.float() # [B, 3] | |
| num_present = present_f.sum(dim=-1, keepdim=True).clamp(min=1.0) | |
| # Floor: present modalities each get at least min_w, absent ones get 0. | |
| # Renormalize so weights still sum to 1. | |
| floored = weights * present_f # zero out absent modalities | |
| floor_values = min_w * present_f # minimum per present modality | |
| # Total floor mass consumed by floor | |
| floor_sum = floor_values.sum(dim=-1, keepdim=True) # min_w * num_present | |
| # Remaining mass to distribute via learned weights above the floor | |
| remaining = (1.0 - floor_sum).clamp(min=0.0) | |
| # Residual learned weights (above floor), re-normalised | |
| residual_weights = (floored - floor_values).clamp(min=0.0) | |
| residual_sum = residual_weights.sum(dim=-1, keepdim=True).clamp(min=1e-8) | |
| residual_norm = residual_weights / residual_sum * remaining | |
| weights = floor_values + residual_norm | |
| # Fallback: if all modalities absent, use uniform | |
| no_present_rows = (present_f.sum(dim=-1) < 1e-6) | |
| if no_present_rows.any(): | |
| fallback = torch.ones_like(weights) / 3.0 | |
| weights[no_present_rows] = fallback[no_present_rows] | |
| else: | |
| # No presence mask — apply floor uniformly across all 3 | |
| floor_values = torch.full_like(weights, min_w) | |
| remaining = 1.0 - min_w * 3 | |
| residual = (weights - floor_values).clamp(min=0.0) | |
| residual_sum = residual.sum(dim=-1, keepdim=True).clamp(min=1e-8) | |
| residual_norm = residual / residual_sum * remaining | |
| weights = floor_values + residual_norm | |
| # Compute entropy (max = log(3)≈1.099 for uniform distribution) | |
| entropy = -(weights * torch.log(weights + 1e-8)).sum(dim=-1).mean() | |
| return weights, entropy | |
| class ModalityConfidence(nn.Module): | |
| """Per-modality confidence estimation for uncertainty awareness.""" | |
| def __init__(self, embed_dim: int, hidden_dim: int = 128, dropout: float = 0.1): | |
| super().__init__() | |
| self.audio_confidence = nn.Sequential( | |
| nn.Linear(embed_dim, hidden_dim), | |
| nn.GELU(), | |
| nn.Dropout(dropout), | |
| nn.Linear(hidden_dim, 1), | |
| nn.Sigmoid(), | |
| ) | |
| self.video_confidence = nn.Sequential( | |
| nn.Linear(embed_dim, hidden_dim), | |
| nn.GELU(), | |
| nn.Dropout(dropout), | |
| nn.Linear(hidden_dim, 1), | |
| nn.Sigmoid(), | |
| ) | |
| self.text_confidence = nn.Sequential( | |
| nn.Linear(embed_dim, hidden_dim), | |
| nn.GELU(), | |
| nn.Dropout(dropout), | |
| nn.Linear(hidden_dim, 1), | |
| nn.Sigmoid(), | |
| ) | |
| def forward( | |
| self, | |
| audio_repr: torch.Tensor, | |
| video_repr: torch.Tensor, | |
| text_repr: torch.Tensor, | |
| ) -> torch.Tensor: | |
| """Compute confidence scores for each modality. | |
| Returns: | |
| Confidence scores [batch_size, 3] | |
| """ | |
| audio_conf = self.audio_confidence(audio_repr).squeeze(-1) # [batch_size] | |
| video_conf = self.video_confidence(video_repr).squeeze(-1) | |
| text_conf = self.text_confidence(text_repr).squeeze(-1) | |
| return torch.stack([audio_conf, video_conf, text_conf], dim=-1) # [batch_size, 3] | |
| class MultimodalFusion(nn.Module): | |
| """Enhanced multimodal fusion with entropy regularization and confidence scores. | |
| Supports multiple fusion strategies: | |
| - early: Concatenate all modalities before fusion | |
| - late: Gate modalities then concatenate with individual representations | |
| - hybrid: Use cross-modal attention then gating (default, best for small datasets) | |
| """ | |
| def __init__( | |
| self, | |
| embed_dim: int, | |
| fusion_dim: int, | |
| mode: str = "hybrid", | |
| dropout: float = 0.2, | |
| num_attention_heads: int = 4, | |
| entropy_weight: float = 0.1, | |
| ): | |
| """Initialize multimodal fusion. | |
| Args: | |
| embed_dim: Individual modality embedding dimension | |
| fusion_dim: Fusion output dimension | |
| mode: Fusion strategy ('early', 'late', 'hybrid') | |
| dropout: Dropout rate | |
| num_attention_heads: Number of attention heads | |
| entropy_weight: Weight for entropy regularization (encourages diversity) | |
| """ | |
| super().__init__() | |
| self.mode = mode | |
| self.entropy_weight = entropy_weight | |
| # Gating mechanism (no min_gate floor - entropy regularization handles balance) | |
| self.gating = ModalityGating( | |
| feature_dim=embed_dim, | |
| hidden_dim=fusion_dim, | |
| dropout=dropout, | |
| entropy_weight=entropy_weight, | |
| ) | |
| # Cross-modal attention (optional, helps with hybrid mode) | |
| self.cross_attention = CrossModalAttention( | |
| embed_dim, | |
| num_heads=num_attention_heads, | |
| dropout=dropout, | |
| ) | |
| # Confidence scoring | |
| self.confidence = ModalityConfidence(embed_dim, hidden_dim=fusion_dim, dropout=dropout) | |
| # Determine fusion projection input dimension | |
| if mode == "early": | |
| projection_input_dim = embed_dim * 3 | |
| else: # late or hybrid | |
| projection_input_dim = embed_dim * 4 # gated_fusion + 3 individual reps | |
| # Fusion projection | |
| self.fusion_projection = nn.Sequential( | |
| nn.Linear(projection_input_dim, fusion_dim), | |
| nn.GELU(), | |
| nn.Dropout(dropout), | |
| nn.LayerNorm(fusion_dim), | |
| ) | |
| # Final fusion head | |
| self.fusion_head = nn.Sequential( | |
| nn.Linear(fusion_dim, fusion_dim), | |
| nn.GELU(), | |
| nn.Dropout(dropout), | |
| nn.LayerNorm(fusion_dim), | |
| ) | |
| def forward( | |
| self, | |
| audio_repr: torch.Tensor, | |
| video_repr: torch.Tensor, | |
| text_repr: torch.Tensor, | |
| modality_present: torch.Tensor | None = None, | |
| ) -> tuple[torch.Tensor, dict[str, torch.Tensor], torch.Tensor, torch.Tensor]: | |
| """Perform multimodal fusion. | |
| Args: | |
| audio_repr: Audio representation [batch_size, embed_dim] | |
| video_repr: Video representation [batch_size, embed_dim] | |
| text_repr: Text representation [batch_size, embed_dim] | |
| modality_present: [batch_size, 3] boolean mask of present modalities | |
| Returns: | |
| Tuple of: | |
| - fused: Fused representation [batch_size, fusion_dim] | |
| - modality_scores: Dict with per-modality scores | |
| - entropy: Entropy regularization loss | |
| - confidence: Per-modality confidence [batch_size, 3] | |
| """ | |
| modality_reprs = [audio_repr, video_repr, text_repr] | |
| # Compute gating weights with entropy regularization | |
| weights, entropy = self.gating(modality_reprs, modality_present) | |
| # Compute per-modality confidence scores | |
| confidence = self.confidence(audio_repr, video_repr, text_repr) | |
| # Apply confidence-aware gating (emphasize confident modalities) | |
| weighted_confidence = confidence * weights | |
| confidence_sum = weighted_confidence.sum(dim=-1, keepdim=True).clamp(min=1e-8) | |
| confidence_aware_weights = weighted_confidence / confidence_sum | |
| modality_scores = { | |
| "audio": weights[:, 0].detach(), | |
| "video": weights[:, 1].detach(), | |
| "text": weights[:, 2].detach(), | |
| "audio_confidence": confidence[:, 0].detach(), | |
| "video_confidence": confidence[:, 1].detach(), | |
| "text_confidence": confidence[:, 2].detach(), | |
| # Non-detached weights so gate_balance_loss can backprop | |
| "raw_weights": weights, | |
| } | |
| # Apply fusion strategy | |
| if self.mode == "early": | |
| # Early fusion: just concatenate | |
| fused_input = torch.cat(modality_reprs, dim=-1) | |
| fused = self.fusion_projection(fused_input) | |
| elif self.mode == "late": | |
| # Late fusion: gate then concatenate | |
| gated = ( | |
| audio_repr * weights[:, 0:1] | |
| + video_repr * weights[:, 1:2] | |
| + text_repr * weights[:, 2:3] | |
| ) | |
| fused_input = torch.cat([gated, audio_repr, video_repr, text_repr], dim=-1) | |
| fused = self.fusion_projection(fused_input) | |
| else: # hybrid (default) | |
| # Hybrid: cross-modal attention + gating | |
| # Stack modalities for attention | |
| tokens = torch.stack(modality_reprs, dim=1) # [batch_size, 3, embed_dim] | |
| # Create attention mask if modalities are missing | |
| key_padding_mask = None | |
| if modality_present is not None: | |
| key_padding_mask = ~modality_present # [batch_size, 3] | |
| # Apply cross-modal attention | |
| attended = self.cross_attention(tokens, key_padding_mask=key_padding_mask) | |
| # Apply gating weights to attended tokens | |
| gated = attended * weights.unsqueeze(-1) | |
| gated_fusion = gated.sum(dim=1) # [batch_size, embed_dim] | |
| # Concatenate gated fusion with individual representations | |
| fused_input = torch.cat([gated_fusion, audio_repr, video_repr, text_repr], dim=-1) | |
| fused = self.fusion_projection(fused_input) | |
| # Apply final fusion head | |
| fused = self.fusion_head(fused) | |
| return fused, modality_scores, entropy, confidence |