import math import torch import torch.nn as nn import torchvision.models as models class EncoderCNN(nn.Module): """CNN-based encoder that extracts spatial visual features from images.""" def __init__(self, embed_size=512, freeze_backbone=True, layers_to_unfreeze=0): super().__init__() resnet = models.resnet50(weights=models.ResNet50_Weights.DEFAULT) # remove final pooling and fully connected layers self.cnn_backbone = nn.Sequential(*list(resnet.children())[:-2]) # add projection layer to get desired embedding size self.conv = nn.Conv2d(2048, embed_size, kernel_size=1) # freeze backbone parameters if param says if freeze_backbone: for param in self.cnn_backbone.parameters(): param.requires_grad = False if layers_to_unfreeze > 0: layers = list(self.cnn_backbone.children()) for i in range(min(layers_to_unfreeze, len(layers))): layer_idx = len(layers) - 1 - i for param in layers[layer_idx].parameters(): param.requires_grad = True def forward(self, images): """ Extract visual features from images. Args: images: Batch of images [B, 3, H, W] Returns: features: Sequence of visual features [S, B, embed_size] """ # feature map from CNN backbone feat_map = self.cnn_backbone(images) # [B, 2048, H', W'] # project to embedding dimension emb_map = self.conv(feat_map) # [B, embed_size, H', W'] # reshape to sequence of features B, D, H, W = emb_map.shape features = emb_map.view(B, D, H * W).permute(2, 0, 1) # [H'*W', B, embed_size] return features class CaptionTransformer(nn.Module): """Transformer-based decoder for generating captions from image features.""" def __init__( self, vocab_size, d_model=512, nhead=8, num_layers=6, dim_feedforward=2048, dropout=0.1, max_len=100 ): super().__init__() self.d_model = d_model self.max_len = max_len # token embedding layer self.token_emb = nn.Embedding(vocab_size, d_model) # positional encoding (learnable) self.pos_emb = nn.Parameter(torch.zeros(max_len, 1, d_model)) # transformer decoder layers decoder_layer = nn.TransformerDecoderLayer( d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward, dropout=dropout, batch_first=False ) self.transformer_decoder = nn.TransformerDecoder( decoder_layer=decoder_layer, num_layers=num_layers ) # output projection to vocabulary self.output_proj = nn.Linear(d_model, vocab_size) # initialize params self._init_parameters() def _init_parameters(self): """Initialize model parameters.""" for p in self.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) def forward(self, tgt_sequence, img_memory): """ Perform transformer decoding. Args: tgt_sequence: Input token sequence [T, B] img_memory: Encoded image features [S, B, D] Returns: logits: Output logits [T, B, vocab_size] """ seq_len = tgt_sequence.shape[0] # embed tokens and add positional encoding tgt_emb = self.token_emb(tgt_sequence) * math.sqrt(self.d_model) tgt_emb = tgt_emb + self.pos_emb[:seq_len] # causal mask tgt_mask = nn.Transformer.generate_square_subsequent_mask(seq_len).to(tgt_sequence.device) # decode decoded = self.transformer_decoder( tgt=tgt_emb, memory=img_memory, tgt_mask=tgt_mask ) # project to vocabulary logits = self.output_proj(decoded) return logits class ImageCaptioningModel(nn.Module): """End-to-end image captioning model.""" def __init__(self, encoder, decoder): super().__init__() self.encoder = encoder self.decoder = decoder def forward(self, images, captions): """ Forward pass for training. Args: images: Batch of images [B, 3, H, W] captions: Target captions [B, T] Returns: logits: Output logits [T-1, B, vocab_size] """ # extract image features img_features = self.encoder(images) # [S, B, embed_size] # prepare target sequence for decoder (remove last token) tgt_input = captions[:, :-1].t() # [T-1, B] # generate logits output_logits = self.decoder(tgt_input, img_features) return output_logits def generate(self, image, vocab, max_len=20, beam_size=5, temperature=1.0): """ Generate caption for a single image. Args: image: Image tensor [1, 3, H, W] vocab: Vocabulary object max_len: Maximum caption length beam_size: Size of beam for beam search (1 for greedy) temperature: Sampling temperature (lower = more deterministic) Returns: caption: List of tokens """ self.eval() with torch.no_grad(): # get image features img_features = self.encoder(image) # [S, 1, embed_size] # initialize with token sos_idx = vocab.stoi[""] eos_idx = vocab.stoi[""] if beam_size <= 1: # greedy search curr_tokens = [sos_idx] for _ in range(max_len - 1): # convert tokens to tensor curr_tensor = torch.LongTensor(curr_tokens).unsqueeze(1).to(image.device) # get next token prediction logits = self.decoder(curr_tensor, img_features) next_token_logits = logits[-1, 0, :] if temperature <= 0: # greedy selection next_token = next_token_logits.argmax().item() else: # sample with temperature probs = torch.softmax(next_token_logits / temperature, dim=0) next_token = torch.multinomial(probs, 1).item() # add to caption curr_tokens.append(next_token) # stop if EOS if next_token == eos_idx: break return [vocab.itos[idx] for idx in curr_tokens] else: # beam search beam = [(torch.tensor([[sos_idx]], device=image.device), 0.0)] # List of (sequence_tensor, score) complete_seqs = [] # List of (sequence_tensor, score) active_beams = beam_size # Keep track of active beams to potentially reduce k in topk for _ in range(max_len - 1): candidates = [] for seq_tensor, score in beam: # check if sequence ended if seq_tensor[0, -1].item() == eos_idx: # Normalize score by length (optional, but good practice) normalized_score = score / seq_tensor.shape[1] complete_seqs.append((seq_tensor, normalized_score)) active_beams = max(1, active_beams -1) # Reduce active beams if one completed continue # get predictions # Ensure seq_tensor is correctly shaped for the decoder [T, B] # Current seq_tensor is [1, Current_Seq_Len], so .t() makes it [Current_Seq_Len, 1] logits = self.decoder(seq_tensor.t(), img_features) next_logits = logits[-1, 0, :] # Logits for the last token in the sequence # top k candidates (use current active_beams for k) if temperature <= 0: log_probs = torch.log_softmax(next_logits, dim=0) else: # apply temperature log_probs = torch.log_softmax(next_logits / temperature, dim=0) # topk_probs are log_probs, topk_idx are indices topk_log_probs, topk_idx = log_probs.topk(min(beam_size, active_beams)) # create new candidates for i in range(topk_log_probs.size(0)): # Iterate up to actual number of top k found new_seq = torch.cat([ seq_tensor, torch.tensor([[topk_idx[i].item()]], device=image.device) ], dim=1) # New score is sum of log_probs new_score = score + topk_log_probs[i].item() candidates.append((new_seq, new_score)) if not candidates: # No more candidates to expand break # sort candidates by score and select top beam_size candidates.sort(key=lambda x: x[1], reverse=True) beam = candidates[:beam_size] # If all remaining beams are complete, can break early if all(b[0][0, -1].item() == eos_idx for b in beam): for seq_tensor, score in beam: if not any(torch.equal(seq_tensor, cs[0]) for cs in complete_seqs): # Avoid duplicates normalized_score = score / seq_tensor.shape[1] complete_seqs.append((seq_tensor, normalized_score)) break # After loop, if no complete sequences, use the current beams if not complete_seqs: # Normalize scores for current beams as well for seq_tensor, score in beam: normalized_score = score / seq_tensor.shape[1] complete_seqs.append((seq_tensor, normalized_score)) # Sort all collected sequences (completed or from final beam) by score complete_seqs.sort(key=lambda x: x[1], reverse=True) # Convert to list of (list of tokens, score) results = [] for seq_tensor, score in complete_seqs: tokens = [vocab.itos[idx] for idx in seq_tensor[0].tolist()] results.append((tokens, score)) return results # Return all sorted beams