Spaces:
Running
Running
Download model.py from davidyang5/VisualAnnotation: direct link, hf CLI and curl.
- Browser
- Download file 11.7 kB
-
https://huggingface.co/spaces/davidyang5/VisualAnnotation/resolve/main/model.py
- Command line
-
hf download hf://spaces/davidyang5/VisualAnnotation/model.py
-
curl -L -o model.py https://huggingface.co/spaces/davidyang5/VisualAnnotation/resolve/main/model.py
11.7 kB
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torchvision.models as models | |
| class EncoderCNN(nn.Module): | |
| """CNN-based encoder that extracts spatial visual features from images.""" | |
| def __init__(self, embed_size=512, freeze_backbone=True, layers_to_unfreeze=0): | |
| super().__init__() | |
| resnet = models.resnet50(weights=models.ResNet50_Weights.DEFAULT) | |
| # remove final pooling and fully connected layers | |
| self.cnn_backbone = nn.Sequential(*list(resnet.children())[:-2]) | |
| # add projection layer to get desired embedding size | |
| self.conv = nn.Conv2d(2048, embed_size, kernel_size=1) | |
| # freeze backbone parameters if param says | |
| if freeze_backbone: | |
| for param in self.cnn_backbone.parameters(): | |
| param.requires_grad = False | |
| if layers_to_unfreeze > 0: | |
| layers = list(self.cnn_backbone.children()) | |
| for i in range(min(layers_to_unfreeze, len(layers))): | |
| layer_idx = len(layers) - 1 - i | |
| for param in layers[layer_idx].parameters(): | |
| param.requires_grad = True | |
| def forward(self, images): | |
| """ | |
| Extract visual features from images. | |
| Args: | |
| images: Batch of images [B, 3, H, W] | |
| Returns: | |
| features: Sequence of visual features [S, B, embed_size] | |
| """ | |
| # feature map from CNN backbone | |
| feat_map = self.cnn_backbone(images) # [B, 2048, H', W'] | |
| # project to embedding dimension | |
| emb_map = self.conv(feat_map) # [B, embed_size, H', W'] | |
| # reshape to sequence of features | |
| B, D, H, W = emb_map.shape | |
| features = emb_map.view(B, D, H * W).permute(2, 0, 1) # [H'*W', B, embed_size] | |
| return features | |
| class CaptionTransformer(nn.Module): | |
| """Transformer-based decoder for generating captions from image features.""" | |
| def __init__( | |
| self, | |
| vocab_size, | |
| d_model=512, | |
| nhead=8, | |
| num_layers=6, | |
| dim_feedforward=2048, | |
| dropout=0.1, | |
| max_len=100 | |
| ): | |
| super().__init__() | |
| self.d_model = d_model | |
| self.max_len = max_len | |
| # token embedding layer | |
| self.token_emb = nn.Embedding(vocab_size, d_model) | |
| # positional encoding (learnable) | |
| self.pos_emb = nn.Parameter(torch.zeros(max_len, 1, d_model)) | |
| # transformer decoder layers | |
| decoder_layer = nn.TransformerDecoderLayer( | |
| d_model=d_model, | |
| nhead=nhead, | |
| dim_feedforward=dim_feedforward, | |
| dropout=dropout, | |
| batch_first=False | |
| ) | |
| self.transformer_decoder = nn.TransformerDecoder( | |
| decoder_layer=decoder_layer, | |
| num_layers=num_layers | |
| ) | |
| # output projection to vocabulary | |
| self.output_proj = nn.Linear(d_model, vocab_size) | |
| # initialize params | |
| self._init_parameters() | |
| def _init_parameters(self): | |
| """Initialize model parameters.""" | |
| for p in self.parameters(): | |
| if p.dim() > 1: | |
| nn.init.xavier_uniform_(p) | |
| def forward(self, tgt_sequence, img_memory): | |
| """ | |
| Perform transformer decoding. | |
| Args: | |
| tgt_sequence: Input token sequence [T, B] | |
| img_memory: Encoded image features [S, B, D] | |
| Returns: | |
| logits: Output logits [T, B, vocab_size] | |
| """ | |
| seq_len = tgt_sequence.shape[0] | |
| # embed tokens and add positional encoding | |
| tgt_emb = self.token_emb(tgt_sequence) * math.sqrt(self.d_model) | |
| tgt_emb = tgt_emb + self.pos_emb[:seq_len] | |
| # causal mask | |
| tgt_mask = nn.Transformer.generate_square_subsequent_mask(seq_len).to(tgt_sequence.device) | |
| # decode | |
| decoded = self.transformer_decoder( | |
| tgt=tgt_emb, | |
| memory=img_memory, | |
| tgt_mask=tgt_mask | |
| ) | |
| # project to vocabulary | |
| logits = self.output_proj(decoded) | |
| return logits | |
| class ImageCaptioningModel(nn.Module): | |
| """End-to-end image captioning model.""" | |
| def __init__(self, encoder, decoder): | |
| super().__init__() | |
| self.encoder = encoder | |
| self.decoder = decoder | |
| def forward(self, images, captions): | |
| """ | |
| Forward pass for training. | |
| Args: | |
| images: Batch of images [B, 3, H, W] | |
| captions: Target captions [B, T] | |
| Returns: | |
| logits: Output logits [T-1, B, vocab_size] | |
| """ | |
| # extract image features | |
| img_features = self.encoder(images) # [S, B, embed_size] | |
| # prepare target sequence for decoder (remove last token) | |
| tgt_input = captions[:, :-1].t() # [T-1, B] | |
| # generate logits | |
| output_logits = self.decoder(tgt_input, img_features) | |
| return output_logits | |
| def generate(self, image, vocab, max_len=20, beam_size=5, temperature=1.0): | |
| """ | |
| Generate caption for a single image. | |
| Args: | |
| image: Image tensor [1, 3, H, W] | |
| vocab: Vocabulary object | |
| max_len: Maximum caption length | |
| beam_size: Size of beam for beam search (1 for greedy) | |
| temperature: Sampling temperature (lower = more deterministic) | |
| Returns: | |
| caption: List of tokens | |
| """ | |
| self.eval() | |
| with torch.no_grad(): | |
| # get image features | |
| img_features = self.encoder(image) # [S, 1, embed_size] | |
| # initialize with <SOS> token | |
| sos_idx = vocab.stoi["<SOS>"] | |
| eos_idx = vocab.stoi["<EOS>"] | |
| if beam_size <= 1: | |
| # greedy search | |
| curr_tokens = [sos_idx] | |
| for _ in range(max_len - 1): | |
| # convert tokens to tensor | |
| curr_tensor = torch.LongTensor(curr_tokens).unsqueeze(1).to(image.device) | |
| # get next token prediction | |
| logits = self.decoder(curr_tensor, img_features) | |
| next_token_logits = logits[-1, 0, :] | |
| if temperature <= 0: | |
| # greedy selection | |
| next_token = next_token_logits.argmax().item() | |
| else: | |
| # sample with temperature | |
| probs = torch.softmax(next_token_logits / temperature, dim=0) | |
| next_token = torch.multinomial(probs, 1).item() | |
| # add to caption | |
| curr_tokens.append(next_token) | |
| # stop if EOS | |
| if next_token == eos_idx: | |
| break | |
| return [vocab.itos[idx] for idx in curr_tokens] | |
| else: | |
| # beam search | |
| beam = [(torch.tensor([[sos_idx]], device=image.device), 0.0)] # List of (sequence_tensor, score) | |
| complete_seqs = [] # List of (sequence_tensor, score) | |
| active_beams = beam_size # Keep track of active beams to potentially reduce k in topk | |
| for _ in range(max_len - 1): | |
| candidates = [] | |
| for seq_tensor, score in beam: | |
| # check if sequence ended | |
| if seq_tensor[0, -1].item() == eos_idx: | |
| # Normalize score by length (optional, but good practice) | |
| normalized_score = score / seq_tensor.shape[1] | |
| complete_seqs.append((seq_tensor, normalized_score)) | |
| active_beams = max(1, active_beams -1) # Reduce active beams if one completed | |
| continue | |
| # get predictions | |
| # Ensure seq_tensor is correctly shaped for the decoder [T, B] | |
| # Current seq_tensor is [1, Current_Seq_Len], so .t() makes it [Current_Seq_Len, 1] | |
| logits = self.decoder(seq_tensor.t(), img_features) | |
| next_logits = logits[-1, 0, :] # Logits for the last token in the sequence | |
| # top k candidates (use current active_beams for k) | |
| if temperature <= 0: | |
| log_probs = torch.log_softmax(next_logits, dim=0) | |
| else: | |
| # apply temperature | |
| log_probs = torch.log_softmax(next_logits / temperature, dim=0) | |
| # topk_probs are log_probs, topk_idx are indices | |
| topk_log_probs, topk_idx = log_probs.topk(min(beam_size, active_beams)) | |
| # create new candidates | |
| for i in range(topk_log_probs.size(0)): # Iterate up to actual number of top k found | |
| new_seq = torch.cat([ | |
| seq_tensor, | |
| torch.tensor([[topk_idx[i].item()]], device=image.device) | |
| ], dim=1) | |
| # New score is sum of log_probs | |
| new_score = score + topk_log_probs[i].item() | |
| candidates.append((new_seq, new_score)) | |
| if not candidates: # No more candidates to expand | |
| break | |
| # sort candidates by score and select top beam_size | |
| candidates.sort(key=lambda x: x[1], reverse=True) | |
| beam = candidates[:beam_size] | |
| # If all remaining beams are complete, can break early | |
| if all(b[0][0, -1].item() == eos_idx for b in beam): | |
| for seq_tensor, score in beam: | |
| if not any(torch.equal(seq_tensor, cs[0]) for cs in complete_seqs): # Avoid duplicates | |
| normalized_score = score / seq_tensor.shape[1] | |
| complete_seqs.append((seq_tensor, normalized_score)) | |
| break | |
| # After loop, if no complete sequences, use the current beams | |
| if not complete_seqs: | |
| # Normalize scores for current beams as well | |
| for seq_tensor, score in beam: | |
| normalized_score = score / seq_tensor.shape[1] | |
| complete_seqs.append((seq_tensor, normalized_score)) | |
| # Sort all collected sequences (completed or from final beam) by score | |
| complete_seqs.sort(key=lambda x: x[1], reverse=True) | |
| # Convert to list of (list of tokens, score) | |
| results = [] | |
| for seq_tensor, score in complete_seqs: | |
| tokens = [vocab.itos[idx] for idx in seq_tensor[0].tolist()] | |
| results.append((tokens, score)) | |
| return results # Return all sorted beams | |