File size: 3,115 Bytes
405a3f0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
import torch
import torch.nn as nn
from torchvision import models
from transformers import DistilBertModel

class ExactDistilBertResNetModel(nn.Module):
    """
    Multimodal neural network architecture combining DistilBERT (text representation)
    and ResNet18 (image representation) with a metadata feed-forward layer.
    
    NOTE ON ARCHITECTURE CONSTRAINTS:
    --------------------------------
    Although a ResNet18 module is initialized and its weights exist inside the 
    pretrained checkpoint, the forward pass hardcodes a zero tensor placeholder for 
    the ResNet features:
        `other_features = torch.zeros(batch_size, 512, ...)`
        
    This is preserved to guarantee identical prediction behavior without retraining, 
    since the classifier weights corresponding to these dimensions are random initializations 
    that never received gradients during training.
    """
    def __init__(self, num_classes=2):
        super(ExactDistilBertResNetModel, self).__init__()
        
        # DistilBERT component
        self.distilbert = DistilBertModel.from_pretrained('distilbert-base-uncased')
        distilbert_output_dim = 768  # DistilBERT hidden output size
        
        # ResNet component - without the final fully connected classification layer
        self.resnet = models.resnet18(weights=None)
        self.resnet.fc = nn.Identity()
        
        # Metadata linear projection (6 input features -> 32 projection dimensions)
        self.metadata_fc = nn.Linear(6, 32)
        
        # Combined multimodal classifier
        # Inputs: 768 (DistilBERT CLS) + 32 (metadata) + 512 (ResNet image placeholder) = 1312 dimensions
        self.classifier = nn.Sequential(
            nn.Linear(1312, 128),
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(128, num_classes)
        )
    
    def forward(self, input_ids=None, attention_mask=None, metadata=None):
        # 1. Process text with DistilBERT and grab [CLS] token representation
        distilbert_outputs = self.distilbert(
            input_ids=input_ids,
            attention_mask=attention_mask
        )
        text_features = distilbert_outputs.last_hidden_state[:, 0, :]  # Shape: [batch_size, 768]
        
        # 2. Process review-level metadata features
        if metadata is not None:
            metadata_features = self.metadata_fc(metadata)  # Shape: [batch_size, 32]
        else:
            batch_size = text_features.shape[0]
            metadata_features = torch.zeros(batch_size, 32, device=text_features.device)
        
        # 3. Apply the ResNet features placeholder (strictly 0s as in original model checkpoint configuration)
        batch_size = text_features.shape[0]
        other_features = torch.zeros(batch_size, 512, device=text_features.device)
        
        # 4. Concatenate vectors along features dimension
        combined_features = torch.cat([text_features, metadata_features, other_features], dim=1)
        
        # 5. Execute final classification head layers
        logits = self.classifier(combined_features)
        return logits