import torch import torch.nn as nn from torchvision import models from transformers import DistilBertModel class ExactDistilBertResNetModel(nn.Module): """ Multimodal neural network architecture combining DistilBERT (text representation) and ResNet18 (image representation) with a metadata feed-forward layer. NOTE ON ARCHITECTURE CONSTRAINTS: -------------------------------- Although a ResNet18 module is initialized and its weights exist inside the pretrained checkpoint, the forward pass hardcodes a zero tensor placeholder for the ResNet features: `other_features = torch.zeros(batch_size, 512, ...)` This is preserved to guarantee identical prediction behavior without retraining, since the classifier weights corresponding to these dimensions are random initializations that never received gradients during training. """ def __init__(self, num_classes=2): super(ExactDistilBertResNetModel, self).__init__() # DistilBERT component self.distilbert = DistilBertModel.from_pretrained('distilbert-base-uncased') distilbert_output_dim = 768 # DistilBERT hidden output size # ResNet component - without the final fully connected classification layer self.resnet = models.resnet18(weights=None) self.resnet.fc = nn.Identity() # Metadata linear projection (6 input features -> 32 projection dimensions) self.metadata_fc = nn.Linear(6, 32) # Combined multimodal classifier # Inputs: 768 (DistilBERT CLS) + 32 (metadata) + 512 (ResNet image placeholder) = 1312 dimensions self.classifier = nn.Sequential( nn.Linear(1312, 128), nn.ReLU(), nn.Dropout(0.1), nn.Linear(128, num_classes) ) def forward(self, input_ids=None, attention_mask=None, metadata=None): # 1. Process text with DistilBERT and grab [CLS] token representation distilbert_outputs = self.distilbert( input_ids=input_ids, attention_mask=attention_mask ) text_features = distilbert_outputs.last_hidden_state[:, 0, :] # Shape: [batch_size, 768] # 2. Process review-level metadata features if metadata is not None: metadata_features = self.metadata_fc(metadata) # Shape: [batch_size, 32] else: batch_size = text_features.shape[0] metadata_features = torch.zeros(batch_size, 32, device=text_features.device) # 3. Apply the ResNet features placeholder (strictly 0s as in original model checkpoint configuration) batch_size = text_features.shape[0] other_features = torch.zeros(batch_size, 512, device=text_features.device) # 4. Concatenate vectors along features dimension combined_features = torch.cat([text_features, metadata_features, other_features], dim=1) # 5. Execute final classification head layers logits = self.classifier(combined_features) return logits