File size: 11,664 Bytes
02b6b5d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision.models import resnet50, ResNet50_Weights

class ScalePredictor(nn.Module):
    """Predicts importance weights for different scale levels"""
    def __init__(self, in_channels, num_scales=4):
        super().__init__()
        self.num_scales = num_scales
        self.global_pool = nn.AdaptiveAvgPool2d(1)
        self.fc = nn.Sequential(
            nn.Linear(in_channels, in_channels // 4),
            nn.ReLU(),
            nn.Linear(in_channels // 4, num_scales),
            nn.Softmax(dim=1)
        )
    
    def forward(self, x):
        b, c, h, w = x.shape
        pooled = self.global_pool(x).view(b, c)
        weights = self.fc(pooled)
        return weights

class DynamicScaleAwareFPN(nn.Module):
    """Dynamic Scale-Aware Feature Pyramid"""
    def __init__(self, in_channels_list, out_channels=256):
        super().__init__()
        self.num_scales = len(in_channels_list)
        
        # Lateral connections
        self.lateral_convs = nn.ModuleList([
            nn.Conv2d(in_ch, out_channels, 1) for in_ch in in_channels_list
        ])
        
        # Scale predictor
        self.scale_predictor = ScalePredictor(in_channels_list[-1], self.num_scales)
        
    def forward(self, features):
        # features: list of feature maps from different scales
        # Get scale weights
        scale_weights = self.scale_predictor(features[-1])
        
        # Apply lateral connections
        lateral_features = [conv(feat) for conv, feat in zip(self.lateral_convs, features)]
        
        # Weighted combination
        # Upsample all to the largest size
        target_size = lateral_features[0].shape[2:]
        upsampled = []
        for feat in lateral_features:
            if feat.shape[2:] != target_size:
                feat = F.interpolate(feat, size=target_size, mode='bilinear', align_corners=False)
            upsampled.append(feat)
        
        # Weight and combine
        weighted_features = []
        for i, feat in enumerate(upsampled):
            weighted_features.append(feat * scale_weights[:, i].view(-1, 1, 1, 1))
        
        output = sum(weighted_features)
        return output, scale_weights

class GraphAttentionLayer(nn.Module):
    """Graph Attention for label correlation"""
    def __init__(self, in_features, out_features, num_labels):
        super().__init__()
        self.in_features = in_features
        self.out_features = out_features
        self.num_labels = num_labels
        
        self.W = nn.Linear(in_features, out_features, bias=False)
        self.a = nn.Linear(2 * out_features, 1, bias=False)
        self.leakyrelu = nn.LeakyReLU(0.2)
        
    def forward(self, x, adj=None):
        # x: [batch, num_labels, in_features]
        batch_size = x.size(0)

        Wx = self.W(x)  # [batch, num_labels, out_features]

        # Compute attention coefficients
        a_input = self._prepare_attentional_mechanism_input(Wx)
        # a_input: [batch, num_labels, num_labels, 2*out_features]

        e = self.leakyrelu(self.a(a_input))  # [batch, num_labels, num_labels, 1]
        e = e.squeeze(-1)  # [batch, num_labels, num_labels]

        attention = F.softmax(e, dim=2)  # [batch, num_labels, num_labels]

        # Apply attention
        h_prime = torch.bmm(attention, Wx)  # [batch, num_labels, out_features]

        return h_prime, attention
    
    def _prepare_attentional_mechanism_input(self, Wx):
        num_labels = Wx.size(1)

        # Wx: [batch, num_labels, out_features]
        # We need to create pairs: [batch, num_labels, num_labels, 2*out_features]

        # Repeat for all pairs
        Wx_i = Wx.unsqueeze(2).repeat(1, 1, num_labels, 1)  # [batch, num_labels, num_labels, out_features]
        Wx_j = Wx.unsqueeze(1).repeat(1, num_labels, 1, 1)  # [batch, num_labels, num_labels, out_features]

        # Concatenate pairs
        all_combinations = torch.cat([Wx_i, Wx_j], dim=3)  # [batch, num_labels, num_labels, 2*out_features]

        return all_combinations

class CrossLabelSemanticGraph(nn.Module):
    """Cross-Label Semantic Correlation Graph"""
    def __init__(self, feature_dim, num_labels, num_heads=4):
        super().__init__()
        self.num_labels = num_labels
        self.num_heads = num_heads
        
        self.gat_layers = nn.ModuleList([
            GraphAttentionLayer(feature_dim, feature_dim, num_labels)
            for _ in range(num_heads)
        ])
        
        self.output_projection = nn.Linear(feature_dim * num_heads, feature_dim)
        
    def forward(self, label_features):
        # label_features: [batch, num_labels, feature_dim]
        
        multi_head_outputs = []
        attentions = []
        
        for gat in self.gat_layers:
            out, att = gat(label_features)
            multi_head_outputs.append(out)
            attentions.append(att)
        
        # Concatenate multi-head outputs
        concatenated = torch.cat(multi_head_outputs, dim=2)
        output = self.output_projection(concatenated)
        
        return output, attentions

class DeformableAttention(nn.Module):
    """Memory-efficient Deformable Attention with spatial downsampling"""
    def __init__(self, dim, num_heads=8, max_tokens=196):
        super().__init__()
        self.num_heads = num_heads
        self.scale = (dim // num_heads) ** -0.5
        self.max_tokens = max_tokens  # Reduce to 14x14 = 196 tokens max

        self.qkv = nn.Linear(dim, dim * 3)
        self.proj = nn.Linear(dim, dim)

    def forward(self, x):
        B, N, C = x.shape

        # Spatially downsample if needed to reduce memory
        if N > self.max_tokens:
            # Reshape to 2D, apply adaptive pooling, reshape back
            # Assuming square spatial dimensions
            h = w = int(N ** 0.5)
            target_h = target_w = int(self.max_tokens ** 0.5)

            x_2d = x.transpose(1, 2).reshape(B, C, h, w)
            x_2d = F.adaptive_avg_pool2d(x_2d, (target_h, target_w))
            x = x_2d.reshape(B, C, -1).transpose(1, 2)
            N = x.shape[1]

        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
        q, k, v = qkv[0], qkv[1], qkv[2]

        attn = (q @ k.transpose(-2, -1)) * self.scale
        attn = attn.softmax(dim=-1)

        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
        x = self.proj(x)
        return x

class SpectralSpatialFusion(nn.Module):
    """Spectral-Spatial Decoupled Multi-Head Classification"""
    def __init__(self, feature_dim, num_labels):
        super().__init__()
        
        # Spectral branch
        self.spectral_branch = nn.Sequential(
            nn.Linear(feature_dim, feature_dim // 2),
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(feature_dim // 2, num_labels)
        )
        
        # Spatial branch
        self.spatial_branch = nn.Sequential(
            nn.Linear(feature_dim, feature_dim // 2),
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(feature_dim // 2, num_labels)
        )
        
        # Fusion gate
        self.fusion_gate = nn.Sequential(
            nn.Linear(feature_dim, num_labels),
            nn.Sigmoid()
        )
    
    def forward(self, features):
        spectral_out = self.spectral_branch(features)
        spatial_out = self.spatial_branch(features)
        
        gate = self.fusion_gate(features)
        
        fused_output = gate * spectral_out + (1 - gate) * spatial_out
        
        return fused_output, spectral_out, spatial_out, gate

class UncertaintyHead(nn.Module):
    """Evidential uncertainty estimation"""
    def __init__(self, feature_dim, num_labels):
        super().__init__()
        self.num_labels = num_labels
        self.fc = nn.Linear(feature_dim, num_labels)
        
    def forward(self, x):
        evidence = F.softplus(self.fc(x))
        alpha = evidence + 1
        uncertainty = self.num_labels / torch.sum(alpha, dim=1, keepdim=True)
        return evidence, uncertainty

class AMSINet(nn.Module):
    def __init__(self, num_labels, pretrained=True):
        super().__init__()
        self.num_labels = num_labels
        
        # Backbone: ResNet50 as feature extractor
        if pretrained:
            weights = ResNet50_Weights.DEFAULT
            backbone = resnet50(weights=weights)
        else:
            backbone = resnet50()
            
        self.conv1 = backbone.conv1
        self.bn1 = backbone.bn1
        self.relu = backbone.relu
        self.maxpool = backbone.maxpool
        
        self.layer1 = backbone.layer1  # 256 channels
        self.layer2 = backbone.layer2  # 512 channels
        self.layer3 = backbone.layer3  # 1024 channels
        self.layer4 = backbone.layer4  # 2048 channels
        
        # Dynamic Scale-Aware FPN
        self.dsafp = DynamicScaleAwareFPN([256, 512, 1024, 2048], out_channels=256)
        
        # Deformable Attention
        self.deformable_attn = DeformableAttention(dim=256, num_heads=8)
        
        # Global pooling
        self.global_pool = nn.AdaptiveAvgPool2d(1)
        
        # Cross-Label Semantic Correlation Graph
        self.clscg = CrossLabelSemanticGraph(feature_dim=256, num_labels=num_labels, num_heads=4)
        
        # Create label embeddings
        self.label_embeddings = nn.Parameter(torch.randn(num_labels, 256))
        
        # Spectral-Spatial Fusion
        self.spectral_spatial_fusion = SpectralSpatialFusion(256, num_labels)
        
        # Uncertainty Head
        self.uncertainty_head = UncertaintyHead(256, num_labels)
        
    def forward(self, x):
        # Backbone feature extraction
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)
        
        # Multi-scale features
        c2 = self.layer1(x)
        c3 = self.layer2(c2)
        c4 = self.layer3(c3)
        c5 = self.layer4(c4)
        
        # Dynamic FPN
        fpn_features, scale_weights = self.dsafp([c2, c3, c4, c5])
        
        # Flatten for attention
        b, c, h, w = fpn_features.shape
        fpn_flat = fpn_features.view(b, c, -1).permute(0, 2, 1)  # [B, H*W, C]
        
        # Deformable attention
        attn_features = self.deformable_attn(fpn_flat)
        
        # Global pooling
        attn_features = attn_features.mean(dim=1)  # [B, C]
        
        # Prepare label-specific features
        batch_size = x.size(0)
        label_features = self.label_embeddings.unsqueeze(0).expand(batch_size, -1, -1)
        
        # Add global context to each label
        global_context = attn_features.unsqueeze(1).expand(-1, self.num_labels, -1)
        label_features = label_features + global_context
        
        # Cross-Label Graph reasoning
        graph_features, graph_attentions = self.clscg(label_features)
        
        # Aggregate graph features
        aggregated_features = graph_features.mean(dim=1)  # [B, C]
        
        # Spectral-Spatial Fusion
        output, spectral_out, spatial_out, fusion_gate = self.spectral_spatial_fusion(aggregated_features)
        
        # Uncertainty estimation
        evidence, uncertainty = self.uncertainty_head(aggregated_features)
        
        return {
            'logits': output,
            'spectral_logits': spectral_out,
            'spatial_logits': spatial_out,
            'fusion_gate': fusion_gate,
            'uncertainty': uncertainty,
            'scale_weights': scale_weights,
            'graph_attentions': graph_attentions
        }