Spaces:
Sleeping
Sleeping
File size: 3,115 Bytes
405a3f0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | import torch
import torch.nn as nn
from torchvision import models
from transformers import DistilBertModel
class ExactDistilBertResNetModel(nn.Module):
"""
Multimodal neural network architecture combining DistilBERT (text representation)
and ResNet18 (image representation) with a metadata feed-forward layer.
NOTE ON ARCHITECTURE CONSTRAINTS:
--------------------------------
Although a ResNet18 module is initialized and its weights exist inside the
pretrained checkpoint, the forward pass hardcodes a zero tensor placeholder for
the ResNet features:
`other_features = torch.zeros(batch_size, 512, ...)`
This is preserved to guarantee identical prediction behavior without retraining,
since the classifier weights corresponding to these dimensions are random initializations
that never received gradients during training.
"""
def __init__(self, num_classes=2):
super(ExactDistilBertResNetModel, self).__init__()
# DistilBERT component
self.distilbert = DistilBertModel.from_pretrained('distilbert-base-uncased')
distilbert_output_dim = 768 # DistilBERT hidden output size
# ResNet component - without the final fully connected classification layer
self.resnet = models.resnet18(weights=None)
self.resnet.fc = nn.Identity()
# Metadata linear projection (6 input features -> 32 projection dimensions)
self.metadata_fc = nn.Linear(6, 32)
# Combined multimodal classifier
# Inputs: 768 (DistilBERT CLS) + 32 (metadata) + 512 (ResNet image placeholder) = 1312 dimensions
self.classifier = nn.Sequential(
nn.Linear(1312, 128),
nn.ReLU(),
nn.Dropout(0.1),
nn.Linear(128, num_classes)
)
def forward(self, input_ids=None, attention_mask=None, metadata=None):
# 1. Process text with DistilBERT and grab [CLS] token representation
distilbert_outputs = self.distilbert(
input_ids=input_ids,
attention_mask=attention_mask
)
text_features = distilbert_outputs.last_hidden_state[:, 0, :] # Shape: [batch_size, 768]
# 2. Process review-level metadata features
if metadata is not None:
metadata_features = self.metadata_fc(metadata) # Shape: [batch_size, 32]
else:
batch_size = text_features.shape[0]
metadata_features = torch.zeros(batch_size, 32, device=text_features.device)
# 3. Apply the ResNet features placeholder (strictly 0s as in original model checkpoint configuration)
batch_size = text_features.shape[0]
other_features = torch.zeros(batch_size, 512, device=text_features.device)
# 4. Concatenate vectors along features dimension
combined_features = torch.cat([text_features, metadata_features, other_features], dim=1)
# 5. Execute final classification head layers
logits = self.classifier(combined_features)
return logits
|