AMSI-Net / backend /models_arch.py
numanabubakar's picture
Initialize AMSI-Net Multi-label Backend with Explainability
02b6b5d
Raw History Blame Contribute Delete
11.7 kB
import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision.models import resnet50, ResNet50_Weights
class ScalePredictor(nn.Module):
"""Predicts importance weights for different scale levels"""
def __init__(self, in_channels, num_scales=4):
super().__init__()
self.num_scales = num_scales
self.global_pool = nn.AdaptiveAvgPool2d(1)
self.fc = nn.Sequential(
nn.Linear(in_channels, in_channels // 4),
nn.ReLU(),
nn.Linear(in_channels // 4, num_scales),
nn.Softmax(dim=1)
)
def forward(self, x):
b, c, h, w = x.shape
pooled = self.global_pool(x).view(b, c)
weights = self.fc(pooled)
return weights
class DynamicScaleAwareFPN(nn.Module):
"""Dynamic Scale-Aware Feature Pyramid"""
def __init__(self, in_channels_list, out_channels=256):
super().__init__()
self.num_scales = len(in_channels_list)
# Lateral connections
self.lateral_convs = nn.ModuleList([
nn.Conv2d(in_ch, out_channels, 1) for in_ch in in_channels_list
])
# Scale predictor
self.scale_predictor = ScalePredictor(in_channels_list[-1], self.num_scales)
def forward(self, features):
# features: list of feature maps from different scales
# Get scale weights
scale_weights = self.scale_predictor(features[-1])
# Apply lateral connections
lateral_features = [conv(feat) for conv, feat in zip(self.lateral_convs, features)]
# Weighted combination
# Upsample all to the largest size
target_size = lateral_features[0].shape[2:]
upsampled = []
for feat in lateral_features:
if feat.shape[2:] != target_size:
feat = F.interpolate(feat, size=target_size, mode='bilinear', align_corners=False)
upsampled.append(feat)
# Weight and combine
weighted_features = []
for i, feat in enumerate(upsampled):
weighted_features.append(feat * scale_weights[:, i].view(-1, 1, 1, 1))
output = sum(weighted_features)
return output, scale_weights
class GraphAttentionLayer(nn.Module):
"""Graph Attention for label correlation"""
def __init__(self, in_features, out_features, num_labels):
super().__init__()
self.in_features = in_features
self.out_features = out_features
self.num_labels = num_labels
self.W = nn.Linear(in_features, out_features, bias=False)
self.a = nn.Linear(2 * out_features, 1, bias=False)
self.leakyrelu = nn.LeakyReLU(0.2)
def forward(self, x, adj=None):
# x: [batch, num_labels, in_features]
batch_size = x.size(0)
Wx = self.W(x) # [batch, num_labels, out_features]
# Compute attention coefficients
a_input = self._prepare_attentional_mechanism_input(Wx)
# a_input: [batch, num_labels, num_labels, 2*out_features]
e = self.leakyrelu(self.a(a_input)) # [batch, num_labels, num_labels, 1]
e = e.squeeze(-1) # [batch, num_labels, num_labels]
attention = F.softmax(e, dim=2) # [batch, num_labels, num_labels]
# Apply attention
h_prime = torch.bmm(attention, Wx) # [batch, num_labels, out_features]
return h_prime, attention
def _prepare_attentional_mechanism_input(self, Wx):
num_labels = Wx.size(1)
# Wx: [batch, num_labels, out_features]
# We need to create pairs: [batch, num_labels, num_labels, 2*out_features]
# Repeat for all pairs
Wx_i = Wx.unsqueeze(2).repeat(1, 1, num_labels, 1) # [batch, num_labels, num_labels, out_features]
Wx_j = Wx.unsqueeze(1).repeat(1, num_labels, 1, 1) # [batch, num_labels, num_labels, out_features]
# Concatenate pairs
all_combinations = torch.cat([Wx_i, Wx_j], dim=3) # [batch, num_labels, num_labels, 2*out_features]
return all_combinations
class CrossLabelSemanticGraph(nn.Module):
"""Cross-Label Semantic Correlation Graph"""
def __init__(self, feature_dim, num_labels, num_heads=4):
super().__init__()
self.num_labels = num_labels
self.num_heads = num_heads
self.gat_layers = nn.ModuleList([
GraphAttentionLayer(feature_dim, feature_dim, num_labels)
for _ in range(num_heads)
])
self.output_projection = nn.Linear(feature_dim * num_heads, feature_dim)
def forward(self, label_features):
# label_features: [batch, num_labels, feature_dim]
multi_head_outputs = []
attentions = []
for gat in self.gat_layers:
out, att = gat(label_features)
multi_head_outputs.append(out)
attentions.append(att)
# Concatenate multi-head outputs
concatenated = torch.cat(multi_head_outputs, dim=2)
output = self.output_projection(concatenated)
return output, attentions
class DeformableAttention(nn.Module):
"""Memory-efficient Deformable Attention with spatial downsampling"""
def __init__(self, dim, num_heads=8, max_tokens=196):
super().__init__()
self.num_heads = num_heads
self.scale = (dim // num_heads) ** -0.5
self.max_tokens = max_tokens # Reduce to 14x14 = 196 tokens max
self.qkv = nn.Linear(dim, dim * 3)
self.proj = nn.Linear(dim, dim)
def forward(self, x):
B, N, C = x.shape
# Spatially downsample if needed to reduce memory
if N > self.max_tokens:
# Reshape to 2D, apply adaptive pooling, reshape back
# Assuming square spatial dimensions
h = w = int(N ** 0.5)
target_h = target_w = int(self.max_tokens ** 0.5)
x_2d = x.transpose(1, 2).reshape(B, C, h, w)
x_2d = F.adaptive_avg_pool2d(x_2d, (target_h, target_w))
x = x_2d.reshape(B, C, -1).transpose(1, 2)
N = x.shape[1]
qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
q, k, v = qkv[0], qkv[1], qkv[2]
attn = (q @ k.transpose(-2, -1)) * self.scale
attn = attn.softmax(dim=-1)
x = (attn @ v).transpose(1, 2).reshape(B, N, C)
x = self.proj(x)
return x
class SpectralSpatialFusion(nn.Module):
"""Spectral-Spatial Decoupled Multi-Head Classification"""
def __init__(self, feature_dim, num_labels):
super().__init__()
# Spectral branch
self.spectral_branch = nn.Sequential(
nn.Linear(feature_dim, feature_dim // 2),
nn.ReLU(),
nn.Dropout(0.3),
nn.Linear(feature_dim // 2, num_labels)
)
# Spatial branch
self.spatial_branch = nn.Sequential(
nn.Linear(feature_dim, feature_dim // 2),
nn.ReLU(),
nn.Dropout(0.3),
nn.Linear(feature_dim // 2, num_labels)
)
# Fusion gate
self.fusion_gate = nn.Sequential(
nn.Linear(feature_dim, num_labels),
nn.Sigmoid()
)
def forward(self, features):
spectral_out = self.spectral_branch(features)
spatial_out = self.spatial_branch(features)
gate = self.fusion_gate(features)
fused_output = gate * spectral_out + (1 - gate) * spatial_out
return fused_output, spectral_out, spatial_out, gate
class UncertaintyHead(nn.Module):
"""Evidential uncertainty estimation"""
def __init__(self, feature_dim, num_labels):
super().__init__()
self.num_labels = num_labels
self.fc = nn.Linear(feature_dim, num_labels)
def forward(self, x):
evidence = F.softplus(self.fc(x))
alpha = evidence + 1
uncertainty = self.num_labels / torch.sum(alpha, dim=1, keepdim=True)
return evidence, uncertainty
class AMSINet(nn.Module):
def __init__(self, num_labels, pretrained=True):
super().__init__()
self.num_labels = num_labels
# Backbone: ResNet50 as feature extractor
if pretrained:
weights = ResNet50_Weights.DEFAULT
backbone = resnet50(weights=weights)
else:
backbone = resnet50()
self.conv1 = backbone.conv1
self.bn1 = backbone.bn1
self.relu = backbone.relu
self.maxpool = backbone.maxpool
self.layer1 = backbone.layer1 # 256 channels
self.layer2 = backbone.layer2 # 512 channels
self.layer3 = backbone.layer3 # 1024 channels
self.layer4 = backbone.layer4 # 2048 channels
# Dynamic Scale-Aware FPN
self.dsafp = DynamicScaleAwareFPN([256, 512, 1024, 2048], out_channels=256)
# Deformable Attention
self.deformable_attn = DeformableAttention(dim=256, num_heads=8)
# Global pooling
self.global_pool = nn.AdaptiveAvgPool2d(1)
# Cross-Label Semantic Correlation Graph
self.clscg = CrossLabelSemanticGraph(feature_dim=256, num_labels=num_labels, num_heads=4)
# Create label embeddings
self.label_embeddings = nn.Parameter(torch.randn(num_labels, 256))
# Spectral-Spatial Fusion
self.spectral_spatial_fusion = SpectralSpatialFusion(256, num_labels)
# Uncertainty Head
self.uncertainty_head = UncertaintyHead(256, num_labels)
def forward(self, x):
# Backbone feature extraction
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.maxpool(x)
# Multi-scale features
c2 = self.layer1(x)
c3 = self.layer2(c2)
c4 = self.layer3(c3)
c5 = self.layer4(c4)
# Dynamic FPN
fpn_features, scale_weights = self.dsafp([c2, c3, c4, c5])
# Flatten for attention
b, c, h, w = fpn_features.shape
fpn_flat = fpn_features.view(b, c, -1).permute(0, 2, 1) # [B, H*W, C]
# Deformable attention
attn_features = self.deformable_attn(fpn_flat)
# Global pooling
attn_features = attn_features.mean(dim=1) # [B, C]
# Prepare label-specific features
batch_size = x.size(0)
label_features = self.label_embeddings.unsqueeze(0).expand(batch_size, -1, -1)
# Add global context to each label
global_context = attn_features.unsqueeze(1).expand(-1, self.num_labels, -1)
label_features = label_features + global_context
# Cross-Label Graph reasoning
graph_features, graph_attentions = self.clscg(label_features)
# Aggregate graph features
aggregated_features = graph_features.mean(dim=1) # [B, C]
# Spectral-Spatial Fusion
output, spectral_out, spatial_out, fusion_gate = self.spectral_spatial_fusion(aggregated_features)
# Uncertainty estimation
evidence, uncertainty = self.uncertainty_head(aggregated_features)
return {
'logits': output,
'spectral_logits': spectral_out,
'spatial_logits': spatial_out,
'fusion_gate': fusion_gate,
'uncertainty': uncertainty,
'scale_weights': scale_weights,
'graph_attentions': graph_attentions
}