File size: 11,676 Bytes
b027f7e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
import math
import torch
import torch.nn as nn
import torchvision.models as models

class EncoderCNN(nn.Module):
    """CNN-based encoder that extracts spatial visual features from images."""
    
    def __init__(self, embed_size=512, freeze_backbone=True, layers_to_unfreeze=0):
        super().__init__()
        resnet = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)
        
        # remove final pooling and fully connected layers
        self.cnn_backbone = nn.Sequential(*list(resnet.children())[:-2])
        
        # add projection layer to get desired embedding size
        self.conv = nn.Conv2d(2048, embed_size, kernel_size=1)
        
        # freeze backbone parameters if param says
        if freeze_backbone:
            for param in self.cnn_backbone.parameters():
                param.requires_grad = False
                
            if layers_to_unfreeze > 0:
                layers = list(self.cnn_backbone.children())
                for i in range(min(layers_to_unfreeze, len(layers))):
                    layer_idx = len(layers) - 1 - i
                    for param in layers[layer_idx].parameters():
                        param.requires_grad = True

    def forward(self, images):
        """
        Extract visual features from images.
        
        Args:
            images: Batch of images [B, 3, H, W]
            
        Returns:
            features: Sequence of visual features [S, B, embed_size]
        """
        # feature map from CNN backbone
        feat_map = self.cnn_backbone(images)  # [B, 2048, H', W']
        
        # project to embedding dimension
        emb_map = self.conv(feat_map)  # [B, embed_size, H', W']
        
        # reshape to sequence of features
        B, D, H, W = emb_map.shape
        features = emb_map.view(B, D, H * W).permute(2, 0, 1)  # [H'*W', B, embed_size]
        
        return features

class CaptionTransformer(nn.Module):
    """Transformer-based decoder for generating captions from image features."""
    
    def __init__(
        self, 
        vocab_size, 
        d_model=512, 
        nhead=8, 
        num_layers=6,
        dim_feedforward=2048, 
        dropout=0.1, 
        max_len=100
    ):
        super().__init__()
        
        self.d_model = d_model
        self.max_len = max_len
        
        # token embedding layer
        self.token_emb = nn.Embedding(vocab_size, d_model)
        
        # positional encoding (learnable)
        self.pos_emb = nn.Parameter(torch.zeros(max_len, 1, d_model))
        
        # transformer decoder layers
        decoder_layer = nn.TransformerDecoderLayer(
            d_model=d_model,
            nhead=nhead,
            dim_feedforward=dim_feedforward,
            dropout=dropout,
            batch_first=False
        )
        
        self.transformer_decoder = nn.TransformerDecoder(
            decoder_layer=decoder_layer,
            num_layers=num_layers
        )
        
        # output projection to vocabulary
        self.output_proj = nn.Linear(d_model, vocab_size)
        
        # initialize params
        self._init_parameters()
        
    def _init_parameters(self):
        """Initialize model parameters."""
        for p in self.parameters():
            if p.dim() > 1:
                nn.init.xavier_uniform_(p)

    def forward(self, tgt_sequence, img_memory):
        """
        Perform transformer decoding.
        
        Args:
            tgt_sequence: Input token sequence [T, B]
            img_memory: Encoded image features [S, B, D]
            
        Returns:
            logits: Output logits [T, B, vocab_size]
        """
        seq_len = tgt_sequence.shape[0]
        
        # embed tokens and add positional encoding
        tgt_emb = self.token_emb(tgt_sequence) * math.sqrt(self.d_model)
        tgt_emb = tgt_emb + self.pos_emb[:seq_len]
        
        # causal mask
        tgt_mask = nn.Transformer.generate_square_subsequent_mask(seq_len).to(tgt_sequence.device)
        
        # decode
        decoded = self.transformer_decoder(
            tgt=tgt_emb,
            memory=img_memory,
            tgt_mask=tgt_mask
        )
        
        # project to vocabulary
        logits = self.output_proj(decoded)
        
        return logits

class ImageCaptioningModel(nn.Module):
    """End-to-end image captioning model."""
    
    def __init__(self, encoder, decoder):
        super().__init__()
        self.encoder = encoder
        self.decoder = decoder

    def forward(self, images, captions):
        """
        Forward pass for training.
        
        Args:
            images: Batch of images [B, 3, H, W]
            captions: Target captions [B, T]
            
        Returns:
            logits: Output logits [T-1, B, vocab_size]
        """
        # extract image features
        img_features = self.encoder(images)  # [S, B, embed_size]
        
        # prepare target sequence for decoder (remove last token)
        tgt_input = captions[:, :-1].t()  # [T-1, B]
        
        # generate logits
        output_logits = self.decoder(tgt_input, img_features)
        
        return output_logits
    
    def generate(self, image, vocab, max_len=20, beam_size=5, temperature=1.0):
        """
        Generate caption for a single image.
        
        Args:
            image: Image tensor [1, 3, H, W]
            vocab: Vocabulary object
            max_len: Maximum caption length
            beam_size: Size of beam for beam search (1 for greedy)
            temperature: Sampling temperature (lower = more deterministic)
            
        Returns:
            caption: List of tokens
        """
        self.eval()
        
        with torch.no_grad():
            # get image features
            img_features = self.encoder(image)  # [S, 1, embed_size]
            
            # initialize with <SOS> token
            sos_idx = vocab.stoi["<SOS>"]
            eos_idx = vocab.stoi["<EOS>"]
            
            if beam_size <= 1:
                # greedy search
                curr_tokens = [sos_idx]
                
                for _ in range(max_len - 1):
                    # convert tokens to tensor
                    curr_tensor = torch.LongTensor(curr_tokens).unsqueeze(1).to(image.device)
                    
                    # get next token prediction
                    logits = self.decoder(curr_tensor, img_features)
                    next_token_logits = logits[-1, 0, :]
                    
                    if temperature <= 0:
                        # greedy selection
                        next_token = next_token_logits.argmax().item()
                    else:
                        # sample with temperature
                        probs = torch.softmax(next_token_logits / temperature, dim=0)
                        next_token = torch.multinomial(probs, 1).item()
                    
                    # add to caption
                    curr_tokens.append(next_token)
                    
                    # stop if EOS
                    if next_token == eos_idx:
                        break
                        
                return [vocab.itos[idx] for idx in curr_tokens]
                
            else:
                # beam search
                beam = [(torch.tensor([[sos_idx]], device=image.device), 0.0)]  # List of (sequence_tensor, score)
                complete_seqs = []  # List of (sequence_tensor, score)
                
                active_beams = beam_size # Keep track of active beams to potentially reduce k in topk

                for _ in range(max_len - 1):
                    candidates = []
                    
                    for seq_tensor, score in beam:
                        # check if sequence ended
                        if seq_tensor[0, -1].item() == eos_idx:
                            # Normalize score by length (optional, but good practice)
                            normalized_score = score / seq_tensor.shape[1] 
                            complete_seqs.append((seq_tensor, normalized_score))
                            active_beams = max(1, active_beams -1) # Reduce active beams if one completed
                            continue
                            
                        # get predictions
                        # Ensure seq_tensor is correctly shaped for the decoder [T, B]
                        # Current seq_tensor is [1, Current_Seq_Len], so .t() makes it [Current_Seq_Len, 1]
                        logits = self.decoder(seq_tensor.t(), img_features) 
                        next_logits = logits[-1, 0, :] # Logits for the last token in the sequence
                        
                        # top k candidates (use current active_beams for k)
                        if temperature <= 0:
                            log_probs = torch.log_softmax(next_logits, dim=0)
                        else:
                            # apply temperature
                            log_probs = torch.log_softmax(next_logits / temperature, dim=0)
                            
                        # topk_probs are log_probs, topk_idx are indices
                        topk_log_probs, topk_idx = log_probs.topk(min(beam_size, active_beams)) 
                        
                        # create new candidates
                        for i in range(topk_log_probs.size(0)): # Iterate up to actual number of top k found
                            new_seq = torch.cat([
                                seq_tensor, 
                                torch.tensor([[topk_idx[i].item()]], device=image.device)
                            ], dim=1)
                            # New score is sum of log_probs
                            new_score = score + topk_log_probs[i].item() 
                            candidates.append((new_seq, new_score))
                    
                    if not candidates: # No more candidates to expand
                        break

                    # sort candidates by score and select top beam_size
                    candidates.sort(key=lambda x: x[1], reverse=True)
                    beam = candidates[:beam_size]
                    
                    # If all remaining beams are complete, can break early
                    if all(b[0][0, -1].item() == eos_idx for b in beam):
                        for seq_tensor, score in beam:
                            if not any(torch.equal(seq_tensor, cs[0]) for cs in complete_seqs): # Avoid duplicates
                                normalized_score = score / seq_tensor.shape[1]
                                complete_seqs.append((seq_tensor, normalized_score))
                        break


                # After loop, if no complete sequences, use the current beams
                if not complete_seqs:
                    # Normalize scores for current beams as well
                    for seq_tensor, score in beam:
                        normalized_score = score / seq_tensor.shape[1]
                        complete_seqs.append((seq_tensor, normalized_score))
                
                # Sort all collected sequences (completed or from final beam) by score
                complete_seqs.sort(key=lambda x: x[1], reverse=True)
                
                # Convert to list of (list of tokens, score)
                results = []
                for seq_tensor, score in complete_seqs:
                    tokens = [vocab.itos[idx] for idx in seq_tensor[0].tolist()]
                    results.append((tokens, score))
                
                return results # Return all sorted beams