File size: 13,566 Bytes
8d7b4b1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
from __future__ import annotations

import torch
import torch.nn as nn
import torch.nn.functional as F


class CrossModalAttention(nn.Module):
    """Cross-modal attention mechanism for multimodal fusion.
    
    Allows each modality to attend to other modalities for better integration.
    """
    def __init__(self, embed_dim: int, num_heads: int = 4, dropout: float = 0.1):
        super().__init__()
        self.attention = nn.MultiheadAttention(
            embed_dim=embed_dim,
            num_heads=num_heads,
            dropout=dropout,
            batch_first=True,
        )
        self.proj = nn.Sequential(
            nn.Linear(embed_dim, embed_dim),
            nn.GELU(),
            nn.Dropout(dropout),
            nn.LayerNorm(embed_dim),
        )

    def forward(
        self,
        modality_tokens: torch.Tensor,
        key_padding_mask: torch.Tensor | None = None,
    ) -> torch.Tensor:
        """Apply cross-modal attention.
        
        Args:
            modality_tokens: [batch_size, num_modalities, embed_dim]
            key_padding_mask: [batch_size, num_modalities] boolean mask
            
        Returns:
            Attended tokens [batch_size, num_modalities, embed_dim]
        """
        attended, _ = self.attention(
            modality_tokens,
            modality_tokens,
            modality_tokens,
            key_padding_mask=key_padding_mask,
        )
        return self.proj(attended + modality_tokens)


class ModalityGating(nn.Module):
    """Learnable modality gating without fixed floor constraints.
    
    Learns to weight each modality dynamically while respecting presence masks.
    """
    def __init__(
        self,
        feature_dim: int,
        hidden_dim: int = 256,
        dropout: float = 0.1,
        entropy_weight: float = 0.1,
    ):
        super().__init__()
        self.entropy_weight = entropy_weight
        
        # Per-modality gating networks
        self.modality_heads = nn.ModuleList(
            [
                nn.Sequential(
                    nn.Linear(feature_dim, hidden_dim),
                    nn.GELU(),
                    nn.Dropout(dropout),
                    nn.Linear(hidden_dim, 1),
                )
                for _ in range(3)
            ]
        )

    def forward(
        self,
        modality_reprs: list[torch.Tensor],
        modality_present: torch.Tensor | None = None,
    ) -> tuple[torch.Tensor, torch.Tensor]:
        """Compute gating weights for modalities.
        
        Args:
            modality_reprs: List of [batch_size, embed_dim] tensors for each modality
            modality_present: [batch_size, 3] boolean mask of which modalities are present
            
        Returns:
            Tuple of:
                - weights: [batch_size, 3] normalized weights
                - entropy: scalar entropy regularization loss
        """
        # Compute logits for each modality
        logits = torch.cat(
            [head(mod) for head, mod in zip(self.modality_heads, modality_reprs)],
            dim=-1,  # [batch_size, 3]
        )

        # Apply presence masking before softmax
        if modality_present is not None:
            # Mask out absent modalities
            logits = logits.masked_fill(~modality_present, float("-1e9"))

        # Compute weights via softmax
        weights = F.softmax(logits, dim=-1)  # [batch_size, 3]

        # Soft gate floor: ensure every *present* modality gets at least min_w weight.
        # This prevents modality collapse (e.g. audio gate → 0.99) during training.
        # Text is encouraged toward higher values via the text_gate_weight loss term
        # in the loss function — not through a hard architectural floor, which would
        # limit the residual learning budget and reduce overall CCC.
        min_w = 0.15
        if modality_present is not None:
            present_f = modality_present.float()  # [B, 3]
            num_present = present_f.sum(dim=-1, keepdim=True).clamp(min=1.0)

            # Floor: present modalities each get at least min_w, absent ones get 0.
            # Renormalize so weights still sum to 1.
            floored = weights * present_f  # zero out absent modalities
            floor_values = min_w * present_f  # minimum per present modality
            # Total floor mass consumed by floor
            floor_sum = floor_values.sum(dim=-1, keepdim=True)  # min_w * num_present
            # Remaining mass to distribute via learned weights above the floor
            remaining = (1.0 - floor_sum).clamp(min=0.0)
            # Residual learned weights (above floor), re-normalised
            residual_weights = (floored - floor_values).clamp(min=0.0)
            residual_sum = residual_weights.sum(dim=-1, keepdim=True).clamp(min=1e-8)
            residual_norm = residual_weights / residual_sum * remaining
            weights = floor_values + residual_norm

            # Fallback: if all modalities absent, use uniform
            no_present_rows = (present_f.sum(dim=-1) < 1e-6)
            if no_present_rows.any():
                fallback = torch.ones_like(weights) / 3.0
                weights[no_present_rows] = fallback[no_present_rows]
        else:
            # No presence mask — apply floor uniformly across all 3
            floor_values = torch.full_like(weights, min_w)
            remaining = 1.0 - min_w * 3
            residual = (weights - floor_values).clamp(min=0.0)
            residual_sum = residual.sum(dim=-1, keepdim=True).clamp(min=1e-8)
            residual_norm = residual / residual_sum * remaining
            weights = floor_values + residual_norm

        # Compute entropy (max = log(3)≈1.099 for uniform distribution)
        entropy = -(weights * torch.log(weights + 1e-8)).sum(dim=-1).mean()

        return weights, entropy


class ModalityConfidence(nn.Module):
    """Per-modality confidence estimation for uncertainty awareness."""
    
    def __init__(self, embed_dim: int, hidden_dim: int = 128, dropout: float = 0.1):
        super().__init__()
        self.audio_confidence = nn.Sequential(
            nn.Linear(embed_dim, hidden_dim),
            nn.GELU(),
            nn.Dropout(dropout),
            nn.Linear(hidden_dim, 1),
            nn.Sigmoid(),
        )
        self.video_confidence = nn.Sequential(
            nn.Linear(embed_dim, hidden_dim),
            nn.GELU(),
            nn.Dropout(dropout),
            nn.Linear(hidden_dim, 1),
            nn.Sigmoid(),
        )
        self.text_confidence = nn.Sequential(
            nn.Linear(embed_dim, hidden_dim),
            nn.GELU(),
            nn.Dropout(dropout),
            nn.Linear(hidden_dim, 1),
            nn.Sigmoid(),
        )

    def forward(
        self,
        audio_repr: torch.Tensor,
        video_repr: torch.Tensor,
        text_repr: torch.Tensor,
    ) -> torch.Tensor:
        """Compute confidence scores for each modality.
        
        Returns:
            Confidence scores [batch_size, 3]
        """
        audio_conf = self.audio_confidence(audio_repr).squeeze(-1)  # [batch_size]
        video_conf = self.video_confidence(video_repr).squeeze(-1)
        text_conf = self.text_confidence(text_repr).squeeze(-1)
        
        return torch.stack([audio_conf, video_conf, text_conf], dim=-1)  # [batch_size, 3]


class MultimodalFusion(nn.Module):
    """Enhanced multimodal fusion with entropy regularization and confidence scores.
    
    Supports multiple fusion strategies:
    - early: Concatenate all modalities before fusion
    - late: Gate modalities then concatenate with individual representations
    - hybrid: Use cross-modal attention then gating (default, best for small datasets)
    """

    def __init__(
        self,
        embed_dim: int,
        fusion_dim: int,
        mode: str = "hybrid",
        dropout: float = 0.2,
        num_attention_heads: int = 4,
        entropy_weight: float = 0.1,
    ):
        """Initialize multimodal fusion.
        
        Args:
            embed_dim: Individual modality embedding dimension
            fusion_dim: Fusion output dimension
            mode: Fusion strategy ('early', 'late', 'hybrid')
            dropout: Dropout rate
            num_attention_heads: Number of attention heads
            entropy_weight: Weight for entropy regularization (encourages diversity)
        """
        super().__init__()
        self.mode = mode
        self.entropy_weight = entropy_weight

        # Gating mechanism (no min_gate floor - entropy regularization handles balance)
        self.gating = ModalityGating(
            feature_dim=embed_dim,
            hidden_dim=fusion_dim,
            dropout=dropout,
            entropy_weight=entropy_weight,
        )

        # Cross-modal attention (optional, helps with hybrid mode)
        self.cross_attention = CrossModalAttention(
            embed_dim,
            num_heads=num_attention_heads,
            dropout=dropout,
        )

        # Confidence scoring
        self.confidence = ModalityConfidence(embed_dim, hidden_dim=fusion_dim, dropout=dropout)

        # Determine fusion projection input dimension
        if mode == "early":
            projection_input_dim = embed_dim * 3
        else:  # late or hybrid
            projection_input_dim = embed_dim * 4  # gated_fusion + 3 individual reps

        # Fusion projection
        self.fusion_projection = nn.Sequential(
            nn.Linear(projection_input_dim, fusion_dim),
            nn.GELU(),
            nn.Dropout(dropout),
            nn.LayerNorm(fusion_dim),
        )

        # Final fusion head
        self.fusion_head = nn.Sequential(
            nn.Linear(fusion_dim, fusion_dim),
            nn.GELU(),
            nn.Dropout(dropout),
            nn.LayerNorm(fusion_dim),
        )

    def forward(
        self,
        audio_repr: torch.Tensor,
        video_repr: torch.Tensor,
        text_repr: torch.Tensor,
        modality_present: torch.Tensor | None = None,
    ) -> tuple[torch.Tensor, dict[str, torch.Tensor], torch.Tensor, torch.Tensor]:
        """Perform multimodal fusion.
        
        Args:
            audio_repr: Audio representation [batch_size, embed_dim]
            video_repr: Video representation [batch_size, embed_dim]
            text_repr: Text representation [batch_size, embed_dim]
            modality_present: [batch_size, 3] boolean mask of present modalities
            
        Returns:
            Tuple of:
                - fused: Fused representation [batch_size, fusion_dim]
                - modality_scores: Dict with per-modality scores
                - entropy: Entropy regularization loss
                - confidence: Per-modality confidence [batch_size, 3]
        """
        modality_reprs = [audio_repr, video_repr, text_repr]

        # Compute gating weights with entropy regularization
        weights, entropy = self.gating(modality_reprs, modality_present)

        # Compute per-modality confidence scores
        confidence = self.confidence(audio_repr, video_repr, text_repr)

        # Apply confidence-aware gating (emphasize confident modalities)
        weighted_confidence = confidence * weights
        confidence_sum = weighted_confidence.sum(dim=-1, keepdim=True).clamp(min=1e-8)
        confidence_aware_weights = weighted_confidence / confidence_sum

        modality_scores = {
            "audio": weights[:, 0].detach(),
            "video": weights[:, 1].detach(),
            "text": weights[:, 2].detach(),
            "audio_confidence": confidence[:, 0].detach(),
            "video_confidence": confidence[:, 1].detach(),
            "text_confidence": confidence[:, 2].detach(),
            # Non-detached weights so gate_balance_loss can backprop
            "raw_weights": weights,
        }

        # Apply fusion strategy
        if self.mode == "early":
            # Early fusion: just concatenate
            fused_input = torch.cat(modality_reprs, dim=-1)
            fused = self.fusion_projection(fused_input)

        elif self.mode == "late":
            # Late fusion: gate then concatenate
            gated = (
                audio_repr * weights[:, 0:1]
                + video_repr * weights[:, 1:2]
                + text_repr * weights[:, 2:3]
            )
            fused_input = torch.cat([gated, audio_repr, video_repr, text_repr], dim=-1)
            fused = self.fusion_projection(fused_input)

        else:  # hybrid (default)
            # Hybrid: cross-modal attention + gating
            # Stack modalities for attention
            tokens = torch.stack(modality_reprs, dim=1)  # [batch_size, 3, embed_dim]
            
            # Create attention mask if modalities are missing
            key_padding_mask = None
            if modality_present is not None:
                key_padding_mask = ~modality_present  # [batch_size, 3]

            # Apply cross-modal attention
            attended = self.cross_attention(tokens, key_padding_mask=key_padding_mask)

            # Apply gating weights to attended tokens
            gated = attended * weights.unsqueeze(-1)
            gated_fusion = gated.sum(dim=1)  # [batch_size, embed_dim]

            # Concatenate gated fusion with individual representations
            fused_input = torch.cat([gated_fusion, audio_repr, video_repr, text_repr], dim=-1)
            fused = self.fusion_projection(fused_input)

        # Apply final fusion head
        fused = self.fusion_head(fused)

        return fused, modality_scores, entropy, confidence