VisualAnnotation / model.py
davidyang5's picture
init
b027f7e
Raw History Blame Contribute Delete
11.7 kB
import math
import torch
import torch.nn as nn
import torchvision.models as models
class EncoderCNN(nn.Module):
"""CNN-based encoder that extracts spatial visual features from images."""
def __init__(self, embed_size=512, freeze_backbone=True, layers_to_unfreeze=0):
super().__init__()
resnet = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)
# remove final pooling and fully connected layers
self.cnn_backbone = nn.Sequential(*list(resnet.children())[:-2])
# add projection layer to get desired embedding size
self.conv = nn.Conv2d(2048, embed_size, kernel_size=1)
# freeze backbone parameters if param says
if freeze_backbone:
for param in self.cnn_backbone.parameters():
param.requires_grad = False
if layers_to_unfreeze > 0:
layers = list(self.cnn_backbone.children())
for i in range(min(layers_to_unfreeze, len(layers))):
layer_idx = len(layers) - 1 - i
for param in layers[layer_idx].parameters():
param.requires_grad = True
def forward(self, images):
"""
Extract visual features from images.
Args:
images: Batch of images [B, 3, H, W]
Returns:
features: Sequence of visual features [S, B, embed_size]
"""
# feature map from CNN backbone
feat_map = self.cnn_backbone(images) # [B, 2048, H', W']
# project to embedding dimension
emb_map = self.conv(feat_map) # [B, embed_size, H', W']
# reshape to sequence of features
B, D, H, W = emb_map.shape
features = emb_map.view(B, D, H * W).permute(2, 0, 1) # [H'*W', B, embed_size]
return features
class CaptionTransformer(nn.Module):
"""Transformer-based decoder for generating captions from image features."""
def __init__(
self,
vocab_size,
d_model=512,
nhead=8,
num_layers=6,
dim_feedforward=2048,
dropout=0.1,
max_len=100
):
super().__init__()
self.d_model = d_model
self.max_len = max_len
# token embedding layer
self.token_emb = nn.Embedding(vocab_size, d_model)
# positional encoding (learnable)
self.pos_emb = nn.Parameter(torch.zeros(max_len, 1, d_model))
# transformer decoder layers
decoder_layer = nn.TransformerDecoderLayer(
d_model=d_model,
nhead=nhead,
dim_feedforward=dim_feedforward,
dropout=dropout,
batch_first=False
)
self.transformer_decoder = nn.TransformerDecoder(
decoder_layer=decoder_layer,
num_layers=num_layers
)
# output projection to vocabulary
self.output_proj = nn.Linear(d_model, vocab_size)
# initialize params
self._init_parameters()
def _init_parameters(self):
"""Initialize model parameters."""
for p in self.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
def forward(self, tgt_sequence, img_memory):
"""
Perform transformer decoding.
Args:
tgt_sequence: Input token sequence [T, B]
img_memory: Encoded image features [S, B, D]
Returns:
logits: Output logits [T, B, vocab_size]
"""
seq_len = tgt_sequence.shape[0]
# embed tokens and add positional encoding
tgt_emb = self.token_emb(tgt_sequence) * math.sqrt(self.d_model)
tgt_emb = tgt_emb + self.pos_emb[:seq_len]
# causal mask
tgt_mask = nn.Transformer.generate_square_subsequent_mask(seq_len).to(tgt_sequence.device)
# decode
decoded = self.transformer_decoder(
tgt=tgt_emb,
memory=img_memory,
tgt_mask=tgt_mask
)
# project to vocabulary
logits = self.output_proj(decoded)
return logits
class ImageCaptioningModel(nn.Module):
"""End-to-end image captioning model."""
def __init__(self, encoder, decoder):
super().__init__()
self.encoder = encoder
self.decoder = decoder
def forward(self, images, captions):
"""
Forward pass for training.
Args:
images: Batch of images [B, 3, H, W]
captions: Target captions [B, T]
Returns:
logits: Output logits [T-1, B, vocab_size]
"""
# extract image features
img_features = self.encoder(images) # [S, B, embed_size]
# prepare target sequence for decoder (remove last token)
tgt_input = captions[:, :-1].t() # [T-1, B]
# generate logits
output_logits = self.decoder(tgt_input, img_features)
return output_logits
def generate(self, image, vocab, max_len=20, beam_size=5, temperature=1.0):
"""
Generate caption for a single image.
Args:
image: Image tensor [1, 3, H, W]
vocab: Vocabulary object
max_len: Maximum caption length
beam_size: Size of beam for beam search (1 for greedy)
temperature: Sampling temperature (lower = more deterministic)
Returns:
caption: List of tokens
"""
self.eval()
with torch.no_grad():
# get image features
img_features = self.encoder(image) # [S, 1, embed_size]
# initialize with <SOS> token
sos_idx = vocab.stoi["<SOS>"]
eos_idx = vocab.stoi["<EOS>"]
if beam_size <= 1:
# greedy search
curr_tokens = [sos_idx]
for _ in range(max_len - 1):
# convert tokens to tensor
curr_tensor = torch.LongTensor(curr_tokens).unsqueeze(1).to(image.device)
# get next token prediction
logits = self.decoder(curr_tensor, img_features)
next_token_logits = logits[-1, 0, :]
if temperature <= 0:
# greedy selection
next_token = next_token_logits.argmax().item()
else:
# sample with temperature
probs = torch.softmax(next_token_logits / temperature, dim=0)
next_token = torch.multinomial(probs, 1).item()
# add to caption
curr_tokens.append(next_token)
# stop if EOS
if next_token == eos_idx:
break
return [vocab.itos[idx] for idx in curr_tokens]
else:
# beam search
beam = [(torch.tensor([[sos_idx]], device=image.device), 0.0)] # List of (sequence_tensor, score)
complete_seqs = [] # List of (sequence_tensor, score)
active_beams = beam_size # Keep track of active beams to potentially reduce k in topk
for _ in range(max_len - 1):
candidates = []
for seq_tensor, score in beam:
# check if sequence ended
if seq_tensor[0, -1].item() == eos_idx:
# Normalize score by length (optional, but good practice)
normalized_score = score / seq_tensor.shape[1]
complete_seqs.append((seq_tensor, normalized_score))
active_beams = max(1, active_beams -1) # Reduce active beams if one completed
continue
# get predictions
# Ensure seq_tensor is correctly shaped for the decoder [T, B]
# Current seq_tensor is [1, Current_Seq_Len], so .t() makes it [Current_Seq_Len, 1]
logits = self.decoder(seq_tensor.t(), img_features)
next_logits = logits[-1, 0, :] # Logits for the last token in the sequence
# top k candidates (use current active_beams for k)
if temperature <= 0:
log_probs = torch.log_softmax(next_logits, dim=0)
else:
# apply temperature
log_probs = torch.log_softmax(next_logits / temperature, dim=0)
# topk_probs are log_probs, topk_idx are indices
topk_log_probs, topk_idx = log_probs.topk(min(beam_size, active_beams))
# create new candidates
for i in range(topk_log_probs.size(0)): # Iterate up to actual number of top k found
new_seq = torch.cat([
seq_tensor,
torch.tensor([[topk_idx[i].item()]], device=image.device)
], dim=1)
# New score is sum of log_probs
new_score = score + topk_log_probs[i].item()
candidates.append((new_seq, new_score))
if not candidates: # No more candidates to expand
break
# sort candidates by score and select top beam_size
candidates.sort(key=lambda x: x[1], reverse=True)
beam = candidates[:beam_size]
# If all remaining beams are complete, can break early
if all(b[0][0, -1].item() == eos_idx for b in beam):
for seq_tensor, score in beam:
if not any(torch.equal(seq_tensor, cs[0]) for cs in complete_seqs): # Avoid duplicates
normalized_score = score / seq_tensor.shape[1]
complete_seqs.append((seq_tensor, normalized_score))
break
# After loop, if no complete sequences, use the current beams
if not complete_seqs:
# Normalize scores for current beams as well
for seq_tensor, score in beam:
normalized_score = score / seq_tensor.shape[1]
complete_seqs.append((seq_tensor, normalized_score))
# Sort all collected sequences (completed or from final beam) by score
complete_seqs.sort(key=lambda x: x[1], reverse=True)
# Convert to list of (list of tokens, score)
results = []
for seq_tensor, score in complete_seqs:
tokens = [vocab.itos[idx] for idx in seq_tensor[0].tolist()]
results.append((tokens, score))
return results # Return all sorted beams