Spaces:
Sleeping
Sleeping
File size: 13,566 Bytes
8d7b4b1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 | from __future__ import annotations
import torch
import torch.nn as nn
import torch.nn.functional as F
class CrossModalAttention(nn.Module):
"""Cross-modal attention mechanism for multimodal fusion.
Allows each modality to attend to other modalities for better integration.
"""
def __init__(self, embed_dim: int, num_heads: int = 4, dropout: float = 0.1):
super().__init__()
self.attention = nn.MultiheadAttention(
embed_dim=embed_dim,
num_heads=num_heads,
dropout=dropout,
batch_first=True,
)
self.proj = nn.Sequential(
nn.Linear(embed_dim, embed_dim),
nn.GELU(),
nn.Dropout(dropout),
nn.LayerNorm(embed_dim),
)
def forward(
self,
modality_tokens: torch.Tensor,
key_padding_mask: torch.Tensor | None = None,
) -> torch.Tensor:
"""Apply cross-modal attention.
Args:
modality_tokens: [batch_size, num_modalities, embed_dim]
key_padding_mask: [batch_size, num_modalities] boolean mask
Returns:
Attended tokens [batch_size, num_modalities, embed_dim]
"""
attended, _ = self.attention(
modality_tokens,
modality_tokens,
modality_tokens,
key_padding_mask=key_padding_mask,
)
return self.proj(attended + modality_tokens)
class ModalityGating(nn.Module):
"""Learnable modality gating without fixed floor constraints.
Learns to weight each modality dynamically while respecting presence masks.
"""
def __init__(
self,
feature_dim: int,
hidden_dim: int = 256,
dropout: float = 0.1,
entropy_weight: float = 0.1,
):
super().__init__()
self.entropy_weight = entropy_weight
# Per-modality gating networks
self.modality_heads = nn.ModuleList(
[
nn.Sequential(
nn.Linear(feature_dim, hidden_dim),
nn.GELU(),
nn.Dropout(dropout),
nn.Linear(hidden_dim, 1),
)
for _ in range(3)
]
)
def forward(
self,
modality_reprs: list[torch.Tensor],
modality_present: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Compute gating weights for modalities.
Args:
modality_reprs: List of [batch_size, embed_dim] tensors for each modality
modality_present: [batch_size, 3] boolean mask of which modalities are present
Returns:
Tuple of:
- weights: [batch_size, 3] normalized weights
- entropy: scalar entropy regularization loss
"""
# Compute logits for each modality
logits = torch.cat(
[head(mod) for head, mod in zip(self.modality_heads, modality_reprs)],
dim=-1, # [batch_size, 3]
)
# Apply presence masking before softmax
if modality_present is not None:
# Mask out absent modalities
logits = logits.masked_fill(~modality_present, float("-1e9"))
# Compute weights via softmax
weights = F.softmax(logits, dim=-1) # [batch_size, 3]
# Soft gate floor: ensure every *present* modality gets at least min_w weight.
# This prevents modality collapse (e.g. audio gate → 0.99) during training.
# Text is encouraged toward higher values via the text_gate_weight loss term
# in the loss function — not through a hard architectural floor, which would
# limit the residual learning budget and reduce overall CCC.
min_w = 0.15
if modality_present is not None:
present_f = modality_present.float() # [B, 3]
num_present = present_f.sum(dim=-1, keepdim=True).clamp(min=1.0)
# Floor: present modalities each get at least min_w, absent ones get 0.
# Renormalize so weights still sum to 1.
floored = weights * present_f # zero out absent modalities
floor_values = min_w * present_f # minimum per present modality
# Total floor mass consumed by floor
floor_sum = floor_values.sum(dim=-1, keepdim=True) # min_w * num_present
# Remaining mass to distribute via learned weights above the floor
remaining = (1.0 - floor_sum).clamp(min=0.0)
# Residual learned weights (above floor), re-normalised
residual_weights = (floored - floor_values).clamp(min=0.0)
residual_sum = residual_weights.sum(dim=-1, keepdim=True).clamp(min=1e-8)
residual_norm = residual_weights / residual_sum * remaining
weights = floor_values + residual_norm
# Fallback: if all modalities absent, use uniform
no_present_rows = (present_f.sum(dim=-1) < 1e-6)
if no_present_rows.any():
fallback = torch.ones_like(weights) / 3.0
weights[no_present_rows] = fallback[no_present_rows]
else:
# No presence mask — apply floor uniformly across all 3
floor_values = torch.full_like(weights, min_w)
remaining = 1.0 - min_w * 3
residual = (weights - floor_values).clamp(min=0.0)
residual_sum = residual.sum(dim=-1, keepdim=True).clamp(min=1e-8)
residual_norm = residual / residual_sum * remaining
weights = floor_values + residual_norm
# Compute entropy (max = log(3)≈1.099 for uniform distribution)
entropy = -(weights * torch.log(weights + 1e-8)).sum(dim=-1).mean()
return weights, entropy
class ModalityConfidence(nn.Module):
"""Per-modality confidence estimation for uncertainty awareness."""
def __init__(self, embed_dim: int, hidden_dim: int = 128, dropout: float = 0.1):
super().__init__()
self.audio_confidence = nn.Sequential(
nn.Linear(embed_dim, hidden_dim),
nn.GELU(),
nn.Dropout(dropout),
nn.Linear(hidden_dim, 1),
nn.Sigmoid(),
)
self.video_confidence = nn.Sequential(
nn.Linear(embed_dim, hidden_dim),
nn.GELU(),
nn.Dropout(dropout),
nn.Linear(hidden_dim, 1),
nn.Sigmoid(),
)
self.text_confidence = nn.Sequential(
nn.Linear(embed_dim, hidden_dim),
nn.GELU(),
nn.Dropout(dropout),
nn.Linear(hidden_dim, 1),
nn.Sigmoid(),
)
def forward(
self,
audio_repr: torch.Tensor,
video_repr: torch.Tensor,
text_repr: torch.Tensor,
) -> torch.Tensor:
"""Compute confidence scores for each modality.
Returns:
Confidence scores [batch_size, 3]
"""
audio_conf = self.audio_confidence(audio_repr).squeeze(-1) # [batch_size]
video_conf = self.video_confidence(video_repr).squeeze(-1)
text_conf = self.text_confidence(text_repr).squeeze(-1)
return torch.stack([audio_conf, video_conf, text_conf], dim=-1) # [batch_size, 3]
class MultimodalFusion(nn.Module):
"""Enhanced multimodal fusion with entropy regularization and confidence scores.
Supports multiple fusion strategies:
- early: Concatenate all modalities before fusion
- late: Gate modalities then concatenate with individual representations
- hybrid: Use cross-modal attention then gating (default, best for small datasets)
"""
def __init__(
self,
embed_dim: int,
fusion_dim: int,
mode: str = "hybrid",
dropout: float = 0.2,
num_attention_heads: int = 4,
entropy_weight: float = 0.1,
):
"""Initialize multimodal fusion.
Args:
embed_dim: Individual modality embedding dimension
fusion_dim: Fusion output dimension
mode: Fusion strategy ('early', 'late', 'hybrid')
dropout: Dropout rate
num_attention_heads: Number of attention heads
entropy_weight: Weight for entropy regularization (encourages diversity)
"""
super().__init__()
self.mode = mode
self.entropy_weight = entropy_weight
# Gating mechanism (no min_gate floor - entropy regularization handles balance)
self.gating = ModalityGating(
feature_dim=embed_dim,
hidden_dim=fusion_dim,
dropout=dropout,
entropy_weight=entropy_weight,
)
# Cross-modal attention (optional, helps with hybrid mode)
self.cross_attention = CrossModalAttention(
embed_dim,
num_heads=num_attention_heads,
dropout=dropout,
)
# Confidence scoring
self.confidence = ModalityConfidence(embed_dim, hidden_dim=fusion_dim, dropout=dropout)
# Determine fusion projection input dimension
if mode == "early":
projection_input_dim = embed_dim * 3
else: # late or hybrid
projection_input_dim = embed_dim * 4 # gated_fusion + 3 individual reps
# Fusion projection
self.fusion_projection = nn.Sequential(
nn.Linear(projection_input_dim, fusion_dim),
nn.GELU(),
nn.Dropout(dropout),
nn.LayerNorm(fusion_dim),
)
# Final fusion head
self.fusion_head = nn.Sequential(
nn.Linear(fusion_dim, fusion_dim),
nn.GELU(),
nn.Dropout(dropout),
nn.LayerNorm(fusion_dim),
)
def forward(
self,
audio_repr: torch.Tensor,
video_repr: torch.Tensor,
text_repr: torch.Tensor,
modality_present: torch.Tensor | None = None,
) -> tuple[torch.Tensor, dict[str, torch.Tensor], torch.Tensor, torch.Tensor]:
"""Perform multimodal fusion.
Args:
audio_repr: Audio representation [batch_size, embed_dim]
video_repr: Video representation [batch_size, embed_dim]
text_repr: Text representation [batch_size, embed_dim]
modality_present: [batch_size, 3] boolean mask of present modalities
Returns:
Tuple of:
- fused: Fused representation [batch_size, fusion_dim]
- modality_scores: Dict with per-modality scores
- entropy: Entropy regularization loss
- confidence: Per-modality confidence [batch_size, 3]
"""
modality_reprs = [audio_repr, video_repr, text_repr]
# Compute gating weights with entropy regularization
weights, entropy = self.gating(modality_reprs, modality_present)
# Compute per-modality confidence scores
confidence = self.confidence(audio_repr, video_repr, text_repr)
# Apply confidence-aware gating (emphasize confident modalities)
weighted_confidence = confidence * weights
confidence_sum = weighted_confidence.sum(dim=-1, keepdim=True).clamp(min=1e-8)
confidence_aware_weights = weighted_confidence / confidence_sum
modality_scores = {
"audio": weights[:, 0].detach(),
"video": weights[:, 1].detach(),
"text": weights[:, 2].detach(),
"audio_confidence": confidence[:, 0].detach(),
"video_confidence": confidence[:, 1].detach(),
"text_confidence": confidence[:, 2].detach(),
# Non-detached weights so gate_balance_loss can backprop
"raw_weights": weights,
}
# Apply fusion strategy
if self.mode == "early":
# Early fusion: just concatenate
fused_input = torch.cat(modality_reprs, dim=-1)
fused = self.fusion_projection(fused_input)
elif self.mode == "late":
# Late fusion: gate then concatenate
gated = (
audio_repr * weights[:, 0:1]
+ video_repr * weights[:, 1:2]
+ text_repr * weights[:, 2:3]
)
fused_input = torch.cat([gated, audio_repr, video_repr, text_repr], dim=-1)
fused = self.fusion_projection(fused_input)
else: # hybrid (default)
# Hybrid: cross-modal attention + gating
# Stack modalities for attention
tokens = torch.stack(modality_reprs, dim=1) # [batch_size, 3, embed_dim]
# Create attention mask if modalities are missing
key_padding_mask = None
if modality_present is not None:
key_padding_mask = ~modality_present # [batch_size, 3]
# Apply cross-modal attention
attended = self.cross_attention(tokens, key_padding_mask=key_padding_mask)
# Apply gating weights to attended tokens
gated = attended * weights.unsqueeze(-1)
gated_fusion = gated.sum(dim=1) # [batch_size, embed_dim]
# Concatenate gated fusion with individual representations
fused_input = torch.cat([gated_fusion, audio_repr, video_repr, text_repr], dim=-1)
fused = self.fusion_projection(fused_input)
# Apply final fusion head
fused = self.fusion_head(fused)
return fused, modality_scores, entropy, confidence |