davidyang5 commited on
Commit
b027f7e
·
1 Parent(s): a2d80c8
__pycache__/data_utils.cpython-313.pyc ADDED
Binary file (8.73 kB). View file
 
__pycache__/model.cpython-313.pyc ADDED
Binary file (11.9 kB). View file
 
caption_model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b53627f89a4d0ef9170db0a47a30fa18dd0e79a3ce9a1859be19fbef667f142
3
+ size 242022730
data_utils.py ADDED
@@ -0,0 +1,176 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ import pickle
4
+ import torch
5
+ from torch.utils.data import Dataset, DataLoader
6
+ import torchvision.transforms as transforms
7
+ from torchvision.datasets import CocoCaptions
8
+ from collections import Counter
9
+ import nltk
10
+ from nltk.tokenize import word_tokenize
11
+ import random
12
+
13
+ def ensure_nltk_resource(resource_name):
14
+ try:
15
+ nltk.data.find(f'tokenizers/{resource_name}')
16
+ except LookupError:
17
+ nltk.download(resource_name, quiet=True)
18
+
19
+ ensure_nltk_resource('punkt')
20
+
21
+ class Vocabulary:
22
+ """Simple vocabulary wrapper for tokenizing and numericalizing text."""
23
+ def __init__(self, freq_threshold=5):
24
+ self.freq_threshold = freq_threshold
25
+ self.itos = {0: "<PAD>", 1: "<SOS>", 2: "<EOS>", 3: "<UNK>"}
26
+ self.stoi = {v: k for k, v in self.itos.items()}
27
+
28
+ def tokenizer(self, text):
29
+ """Tokenize text to lowercase word tokens."""
30
+ return word_tokenize(text.lower())
31
+
32
+ def build_vocabulary(self, sentence_list):
33
+ """Build vocabulary from list of sentences with frequency threshold."""
34
+ frequencies = Counter()
35
+ idx = 4 # Start after special tokens
36
+
37
+ for sentence in sentence_list:
38
+ tokens = self.tokenizer(sentence)
39
+ frequencies.update(tokens)
40
+
41
+ for word, freq in frequencies.items():
42
+ if freq >= self.freq_threshold:
43
+ self.stoi[word] = idx
44
+ self.itos[idx] = word
45
+ idx += 1
46
+
47
+ def numericalize(self, text):
48
+ """Convert text to token indices."""
49
+ tokenized_text = self.tokenizer(text)
50
+ return [self.stoi.get(token, self.stoi["<UNK>"]) for token in tokenized_text]
51
+
52
+ def __len__(self):
53
+ return len(self.itos)
54
+
55
+ def save(self, path):
56
+ with open(path, "wb") as f:
57
+ pickle.dump(self, f)
58
+
59
+ @classmethod
60
+ def load(cls, path):
61
+ with open(path, "rb") as f:
62
+ return pickle.load(f)
63
+
64
+ def get_transforms(split="train"):
65
+ """Get image transforms for training or validation."""
66
+ normalize = transforms.Normalize(
67
+ mean=[0.485, 0.456, 0.406],
68
+ std=[0.229, 0.224, 0.225]
69
+ )
70
+
71
+ if split == "train":
72
+ return transforms.Compose([
73
+ transforms.Resize((256, 256)),
74
+ transforms.RandomCrop(224),
75
+ transforms.RandomHorizontalFlip(),
76
+ transforms.ToTensor(),
77
+ normalize,
78
+ ])
79
+ else:
80
+ return transforms.Compose([
81
+ transforms.Resize((224, 224)),
82
+ transforms.ToTensor(),
83
+ normalize,
84
+ ])
85
+
86
+ class CaptionCollate:
87
+ """Collate function for DataLoader that handles padding captions."""
88
+ def __init__(self, pad_idx, vocab):
89
+ self.pad_idx = pad_idx
90
+ self.vocab = vocab
91
+
92
+ def __call__(self, batch):
93
+ imgs = torch.stack([item[0] for item in batch], dim=0)
94
+
95
+ captions_batch = []
96
+ for item in batch:
97
+ caption = random.choice(item[1])
98
+ tokens = [self.vocab.stoi["<SOS>"]] + self.vocab.numericalize(caption) + [self.vocab.stoi["<EOS>"]]
99
+ captions_batch.append(torch.tensor(tokens, dtype=torch.long))
100
+
101
+ captions_padded = torch.nn.utils.rnn.pad_sequence(
102
+ captions_batch,
103
+ batch_first=True,
104
+ padding_value=self.pad_idx
105
+ )
106
+
107
+ return imgs, captions_padded
108
+
109
+ def build_or_load_vocab(ann_file, vocab_path="vocab.pkl", freq_threshold=5):
110
+ """Build or load vocabulary from annotations file."""
111
+ if os.path.exists(vocab_path):
112
+ print(f"Loading vocabulary from {vocab_path}")
113
+ return Vocabulary.load(vocab_path)
114
+
115
+ print(f"Building vocabulary from {ann_file}")
116
+ with open(ann_file, 'r') as f:
117
+ annotations = json.load(f)
118
+
119
+ captions = [ann['caption'] for ann in annotations['annotations']]
120
+
121
+ vocab = Vocabulary(freq_threshold=freq_threshold)
122
+ vocab.build_vocabulary(captions)
123
+ vocab.save(vocab_path)
124
+
125
+ return vocab
126
+
127
+ def get_dataloaders(
128
+ train_img_folder, train_ann_file,
129
+ val_img_folder, val_ann_file,
130
+ vocab, batch_size=32, num_workers=4, limit=None
131
+ ):
132
+ """Create data loaders for training and validation."""
133
+ train_transform = get_transforms("train")
134
+ val_transform = get_transforms("val")
135
+
136
+ train_dataset = CocoCaptions(
137
+ root=train_img_folder,
138
+ annFile=train_ann_file,
139
+ transform=train_transform
140
+ )
141
+
142
+ val_dataset = CocoCaptions(
143
+ root=val_img_folder,
144
+ annFile=val_ann_file,
145
+ transform=val_transform
146
+ )
147
+
148
+ if limit:
149
+ train_dataset = torch.utils.data.Subset(
150
+ train_dataset, range(min(limit, len(train_dataset)))
151
+ )
152
+ val_dataset = torch.utils.data.Subset(
153
+ val_dataset, range(min(limit, len(val_dataset)))
154
+ )
155
+
156
+ collate_fn = CaptionCollate(pad_idx=vocab.stoi["<PAD>"], vocab=vocab)
157
+
158
+ train_loader = DataLoader(
159
+ train_dataset,
160
+ batch_size=batch_size,
161
+ shuffle=True,
162
+ num_workers=num_workers,
163
+ collate_fn=collate_fn,
164
+ pin_memory=True,
165
+ )
166
+
167
+ val_loader = DataLoader(
168
+ val_dataset,
169
+ batch_size=batch_size,
170
+ shuffle=False,
171
+ num_workers=num_workers,
172
+ collate_fn=collate_fn,
173
+ pin_memory=True,
174
+ )
175
+
176
+ return train_loader, val_loader
generate.py ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import torch
3
+ import matplotlib
4
+ matplotlib.use('Agg')
5
+ import matplotlib.pyplot as plt
6
+ from PIL import Image
7
+ import torchvision.transforms as transforms
8
+ import random
9
+ import numpy as np
10
+ from torchvision.datasets import CocoCaptions
11
+ import matplotlib.gridspec as gridspec
12
+ import os
13
+ from tqdm import tqdm
14
+
15
+ from model import EncoderCNN, CaptionTransformer, ImageCaptioningModel
16
+ from data_utils import Vocabulary, get_transforms
17
+
18
+ def main():
19
+ parser = argparse.ArgumentParser(description="Generate Image Captions")
20
+
21
+ parser.add_argument("--model_path", type=str, default="caption_model.pt",
22
+ help="Path to trained model (default: caption_model.pt)")
23
+ parser.add_argument("--vocab_path", type=str, default="vocab.pkl",
24
+ help="Path to vocabulary file (default: vocab.pkl)")
25
+
26
+ parser.add_argument("--sample_img_dir", type=str, required=True,
27
+ help="Path to sample images directory")
28
+ parser.add_argument("--sample_ann_file", type=str, default="coco_sample/captions_val_sample.json",
29
+ help="Path to sample annotations JSON file (only used if --sample_img_dir is coco_sample/images)")
30
+
31
+ parser.add_argument("--num_samples", type=int, default=6,
32
+ help="Maximum number of samples to display")
33
+ parser.add_argument("--beam_size", type=int, default=5,
34
+ help="Beam size for beam search")
35
+ parser.add_argument("--temperature", type=float, default=1.0,
36
+ help="Sampling temperature")
37
+ parser.add_argument("--output", type=str,
38
+ help="Path to save output image. If not provided, will use predictions.png")
39
+
40
+ args = parser.parse_args()
41
+
42
+ # Set default output filename
43
+ if args.output is None:
44
+ args.output = "predictions.png"
45
+
46
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
47
+ print(f"Using device: {device}")
48
+
49
+ try:
50
+ vocab = Vocabulary.load(args.vocab_path)
51
+ print(f"Loaded vocabulary with {len(vocab)} tokens from {args.vocab_path}")
52
+ except Exception as e:
53
+ print(f"Error loading vocabulary: {e}")
54
+ return
55
+
56
+ encoder = EncoderCNN(embed_size=512)
57
+ decoder = CaptionTransformer(vocab_size=len(vocab))
58
+ model = ImageCaptioningModel(encoder, decoder)
59
+
60
+ try:
61
+ model.load_state_dict(torch.load(args.model_path, map_location=device))
62
+ model = model.to(device)
63
+ model.eval()
64
+ print(f"Model loaded from {args.model_path}")
65
+ except Exception as e:
66
+ print(f"Error loading model: {e}")
67
+ return
68
+
69
+ model_transform = get_transforms(split="val")
70
+
71
+ image_paths = []
72
+ gt_captions_map = {}
73
+ use_coco_dataset = args.sample_img_dir.startswith("coco_sample")
74
+
75
+ if use_coco_dataset:
76
+ image_root = args.sample_img_dir
77
+ if not image_root.endswith('images'):
78
+ image_root = os.path.join(image_root, 'images')
79
+
80
+ annotation_file = args.sample_ann_file
81
+
82
+ print(f"Attempting to load COCO dataset: images from '{image_root}', annotations from '{annotation_file}'")
83
+
84
+ try:
85
+ if not os.path.isdir(image_root):
86
+ print(f"Error: Image directory not found: {image_root}")
87
+ return
88
+
89
+ sample_dataset = CocoCaptions(
90
+ root=image_root,
91
+ annFile=annotation_file,
92
+ transform=None
93
+ )
94
+ print(f"Loaded COCO sample dataset with {len(sample_dataset)} images initially from annotation file.")
95
+
96
+ available_img_ids = []
97
+ if not sample_dataset.ids:
98
+ print("No image IDs found in the annotation file.")
99
+ return
100
+
101
+ print("Checking for available image files on disk...")
102
+ for img_id in sample_dataset.ids:
103
+ try:
104
+ img_info = sample_dataset.coco.loadImgs(img_id)[0]
105
+ img_file_path = os.path.join(image_root, img_info['file_name'])
106
+ if os.path.exists(img_file_path):
107
+ available_img_ids.append(img_id)
108
+ except Exception as e:
109
+ print(f"Warning: Could not process img_id {img_id} from annotations: {e}")
110
+ continue
111
+
112
+ if not available_img_ids:
113
+ print("No valid image files found on disk corresponding to the annotations.")
114
+ return
115
+
116
+ print(f"Found {len(available_img_ids)} available images on disk.")
117
+
118
+ num_to_sample = min(args.num_samples, 6, len(available_img_ids))
119
+ if num_to_sample == 0:
120
+ print("No images available to sample after checking disk.")
121
+ return
122
+
123
+ selected_img_ids = random.sample(available_img_ids, num_to_sample)
124
+
125
+ for img_id in selected_img_ids:
126
+ img_info = sample_dataset.coco.loadImgs(img_id)[0]
127
+ img_path = os.path.join(image_root, img_info['file_name'])
128
+ image_paths.append(img_path)
129
+
130
+ ann_ids = sample_dataset.coco.getAnnIds(imgIds=img_id)
131
+ anns = sample_dataset.coco.loadAnns(ann_ids)
132
+ gt_caps_for_img = [ann['caption'] for ann in anns]
133
+ if gt_caps_for_img:
134
+ gt_captions_map[img_path] = random.choice(gt_caps_for_img)
135
+ else:
136
+ gt_captions_map[img_path] = "N/A (no captions found for this image)"
137
+
138
+ except FileNotFoundError:
139
+ print(f"Error: Annotation file not found or critical issue during initial load: {annotation_file}")
140
+ return
141
+ except Exception as e:
142
+ print(f"Error loading COCO sample dataset: {e}")
143
+ return
144
+ else:
145
+ print(f"Loading images directly from custom directory: {args.sample_img_dir}")
146
+ try:
147
+ if not os.path.isdir(args.sample_img_dir):
148
+ print(f"Error: Directory not found: {args.sample_img_dir}")
149
+ return
150
+
151
+ all_files = [os.path.join(args.sample_img_dir, f) for f in os.listdir(args.sample_img_dir)]
152
+ image_files = [f for f in all_files if os.path.isfile(f) and f.lower().endswith(('.png', '.jpg', '.jpeg'))]
153
+
154
+ if not image_files:
155
+ print(f"No image files (.png, .jpg, .jpeg) found in {args.sample_img_dir}")
156
+ return
157
+
158
+ print(f"Found {len(image_files)} images.")
159
+ num_available = len(image_files)
160
+ num_samples = min(args.num_samples, 6, num_available)
161
+ image_paths = random.sample(image_files, num_samples)
162
+
163
+ except Exception as e:
164
+ print(f"Error listing or sampling images from {args.sample_img_dir}: {e}")
165
+ return
166
+
167
+ if not image_paths:
168
+ print("No images selected for processing.")
169
+ return
170
+
171
+ num_samples = len(image_paths)
172
+
173
+ if num_samples == 0:
174
+ print("No images to display.")
175
+ return
176
+ elif num_samples <= 3:
177
+ nrows, ncols = 1, num_samples
178
+ else:
179
+ nrows = 2
180
+ ncols = (num_samples + 1) // 2
181
+
182
+ fig = plt.figure(figsize=(5*ncols, 5*nrows))
183
+ gs = gridspec.GridSpec(nrows, ncols, figure=fig)
184
+
185
+ print(f"Generating captions for {num_samples} images...")
186
+
187
+ # Wrap image_paths with tqdm for a progress bar
188
+ for i, img_path in enumerate(tqdm(image_paths, desc="Generating Captions")):
189
+ # Calculate grid position
190
+ row, col = i // ncols, i % ncols
191
+ ax = fig.add_subplot(gs[row, col])
192
+
193
+ try:
194
+ # Load original image using PIL
195
+ original_img = Image.open(img_path).convert("RGB")
196
+ except Exception as e:
197
+ print(f"Error opening image {img_path}: {e}")
198
+ ax.set_title(f"Error loading image: {os.path.basename(img_path)}", fontsize=9)
199
+ ax.axis('off')
200
+ continue # Skip to next image
201
+
202
+ # Apply transform only for model input
203
+ model_input = model_transform(original_img).unsqueeze(0).to(device)
204
+
205
+ # Generate caption
206
+ with torch.no_grad(): # Ensure no gradients are calculated
207
+ # model.generate now returns a list of (tokens_list, score) tuples, sorted by score
208
+ beam_results = model.generate(
209
+ model_input,
210
+ vocab,
211
+ max_len=30,
212
+ beam_size=args.beam_size,
213
+ temperature=args.temperature
214
+ )
215
+
216
+ title_parts = []
217
+
218
+ # Ground Truth
219
+ if use_coco_dataset:
220
+ gt_caption = gt_captions_map.get(img_path, "N/A")
221
+ # Clean GT caption tokens
222
+ gt_caption_cleaned = ' '.join([t for t in gt_caption.split() if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}]) # Assuming gt_caption is a string
223
+ title_parts.append(f"GT: {gt_caption_cleaned}")
224
+
225
+ if beam_results:
226
+ # Best predicted caption
227
+ best_pred_tokens, best_pred_score = beam_results[0]
228
+ best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
229
+ title_parts.append(f"Best Pred (Score: {best_pred_score:.2f}): {best_pred_caption}")
230
+
231
+ # Other beam search results
232
+ if len(beam_results) > 1:
233
+ title_parts.append("--- Other Beams ---")
234
+ for i, (tokens, score) in enumerate(beam_results[1:]):
235
+ if i < args.beam_size -1 : # Show up to beam_size-1 other beams
236
+ other_caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
237
+ title_parts.append(f"Beam {i+1} (Score: {score:.2f}): {other_caption}")
238
+ else:
239
+ break
240
+ else:
241
+ title_parts.append("Pred: No caption generated")
242
+
243
+ # Display original image and set title
244
+ ax.imshow(np.array(original_img)) # Use np.array to ensure correct display
245
+ ax.set_title("\n".join(title_parts), fontsize=8) # Adjust fontsize if needed
246
+
247
+ ax.axis('off')
248
+
249
+ plt.tight_layout(h_pad=3.0)
250
+
251
+ # Save to predictions.png
252
+ plt.savefig(args.output, bbox_inches='tight', dpi=200)
253
+ print(f"Output saved to {args.output}")
254
+
255
+ if __name__ == "__main__":
256
+ main()
model.py ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import torch
3
+ import torch.nn as nn
4
+ import torchvision.models as models
5
+
6
+ class EncoderCNN(nn.Module):
7
+ """CNN-based encoder that extracts spatial visual features from images."""
8
+
9
+ def __init__(self, embed_size=512, freeze_backbone=True, layers_to_unfreeze=0):
10
+ super().__init__()
11
+ resnet = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)
12
+
13
+ # remove final pooling and fully connected layers
14
+ self.cnn_backbone = nn.Sequential(*list(resnet.children())[:-2])
15
+
16
+ # add projection layer to get desired embedding size
17
+ self.conv = nn.Conv2d(2048, embed_size, kernel_size=1)
18
+
19
+ # freeze backbone parameters if param says
20
+ if freeze_backbone:
21
+ for param in self.cnn_backbone.parameters():
22
+ param.requires_grad = False
23
+
24
+ if layers_to_unfreeze > 0:
25
+ layers = list(self.cnn_backbone.children())
26
+ for i in range(min(layers_to_unfreeze, len(layers))):
27
+ layer_idx = len(layers) - 1 - i
28
+ for param in layers[layer_idx].parameters():
29
+ param.requires_grad = True
30
+
31
+ def forward(self, images):
32
+ """
33
+ Extract visual features from images.
34
+
35
+ Args:
36
+ images: Batch of images [B, 3, H, W]
37
+
38
+ Returns:
39
+ features: Sequence of visual features [S, B, embed_size]
40
+ """
41
+ # feature map from CNN backbone
42
+ feat_map = self.cnn_backbone(images) # [B, 2048, H', W']
43
+
44
+ # project to embedding dimension
45
+ emb_map = self.conv(feat_map) # [B, embed_size, H', W']
46
+
47
+ # reshape to sequence of features
48
+ B, D, H, W = emb_map.shape
49
+ features = emb_map.view(B, D, H * W).permute(2, 0, 1) # [H'*W', B, embed_size]
50
+
51
+ return features
52
+
53
+ class CaptionTransformer(nn.Module):
54
+ """Transformer-based decoder for generating captions from image features."""
55
+
56
+ def __init__(
57
+ self,
58
+ vocab_size,
59
+ d_model=512,
60
+ nhead=8,
61
+ num_layers=6,
62
+ dim_feedforward=2048,
63
+ dropout=0.1,
64
+ max_len=100
65
+ ):
66
+ super().__init__()
67
+
68
+ self.d_model = d_model
69
+ self.max_len = max_len
70
+
71
+ # token embedding layer
72
+ self.token_emb = nn.Embedding(vocab_size, d_model)
73
+
74
+ # positional encoding (learnable)
75
+ self.pos_emb = nn.Parameter(torch.zeros(max_len, 1, d_model))
76
+
77
+ # transformer decoder layers
78
+ decoder_layer = nn.TransformerDecoderLayer(
79
+ d_model=d_model,
80
+ nhead=nhead,
81
+ dim_feedforward=dim_feedforward,
82
+ dropout=dropout,
83
+ batch_first=False
84
+ )
85
+
86
+ self.transformer_decoder = nn.TransformerDecoder(
87
+ decoder_layer=decoder_layer,
88
+ num_layers=num_layers
89
+ )
90
+
91
+ # output projection to vocabulary
92
+ self.output_proj = nn.Linear(d_model, vocab_size)
93
+
94
+ # initialize params
95
+ self._init_parameters()
96
+
97
+ def _init_parameters(self):
98
+ """Initialize model parameters."""
99
+ for p in self.parameters():
100
+ if p.dim() > 1:
101
+ nn.init.xavier_uniform_(p)
102
+
103
+ def forward(self, tgt_sequence, img_memory):
104
+ """
105
+ Perform transformer decoding.
106
+
107
+ Args:
108
+ tgt_sequence: Input token sequence [T, B]
109
+ img_memory: Encoded image features [S, B, D]
110
+
111
+ Returns:
112
+ logits: Output logits [T, B, vocab_size]
113
+ """
114
+ seq_len = tgt_sequence.shape[0]
115
+
116
+ # embed tokens and add positional encoding
117
+ tgt_emb = self.token_emb(tgt_sequence) * math.sqrt(self.d_model)
118
+ tgt_emb = tgt_emb + self.pos_emb[:seq_len]
119
+
120
+ # causal mask
121
+ tgt_mask = nn.Transformer.generate_square_subsequent_mask(seq_len).to(tgt_sequence.device)
122
+
123
+ # decode
124
+ decoded = self.transformer_decoder(
125
+ tgt=tgt_emb,
126
+ memory=img_memory,
127
+ tgt_mask=tgt_mask
128
+ )
129
+
130
+ # project to vocabulary
131
+ logits = self.output_proj(decoded)
132
+
133
+ return logits
134
+
135
+ class ImageCaptioningModel(nn.Module):
136
+ """End-to-end image captioning model."""
137
+
138
+ def __init__(self, encoder, decoder):
139
+ super().__init__()
140
+ self.encoder = encoder
141
+ self.decoder = decoder
142
+
143
+ def forward(self, images, captions):
144
+ """
145
+ Forward pass for training.
146
+
147
+ Args:
148
+ images: Batch of images [B, 3, H, W]
149
+ captions: Target captions [B, T]
150
+
151
+ Returns:
152
+ logits: Output logits [T-1, B, vocab_size]
153
+ """
154
+ # extract image features
155
+ img_features = self.encoder(images) # [S, B, embed_size]
156
+
157
+ # prepare target sequence for decoder (remove last token)
158
+ tgt_input = captions[:, :-1].t() # [T-1, B]
159
+
160
+ # generate logits
161
+ output_logits = self.decoder(tgt_input, img_features)
162
+
163
+ return output_logits
164
+
165
+ def generate(self, image, vocab, max_len=20, beam_size=5, temperature=1.0):
166
+ """
167
+ Generate caption for a single image.
168
+
169
+ Args:
170
+ image: Image tensor [1, 3, H, W]
171
+ vocab: Vocabulary object
172
+ max_len: Maximum caption length
173
+ beam_size: Size of beam for beam search (1 for greedy)
174
+ temperature: Sampling temperature (lower = more deterministic)
175
+
176
+ Returns:
177
+ caption: List of tokens
178
+ """
179
+ self.eval()
180
+
181
+ with torch.no_grad():
182
+ # get image features
183
+ img_features = self.encoder(image) # [S, 1, embed_size]
184
+
185
+ # initialize with <SOS> token
186
+ sos_idx = vocab.stoi["<SOS>"]
187
+ eos_idx = vocab.stoi["<EOS>"]
188
+
189
+ if beam_size <= 1:
190
+ # greedy search
191
+ curr_tokens = [sos_idx]
192
+
193
+ for _ in range(max_len - 1):
194
+ # convert tokens to tensor
195
+ curr_tensor = torch.LongTensor(curr_tokens).unsqueeze(1).to(image.device)
196
+
197
+ # get next token prediction
198
+ logits = self.decoder(curr_tensor, img_features)
199
+ next_token_logits = logits[-1, 0, :]
200
+
201
+ if temperature <= 0:
202
+ # greedy selection
203
+ next_token = next_token_logits.argmax().item()
204
+ else:
205
+ # sample with temperature
206
+ probs = torch.softmax(next_token_logits / temperature, dim=0)
207
+ next_token = torch.multinomial(probs, 1).item()
208
+
209
+ # add to caption
210
+ curr_tokens.append(next_token)
211
+
212
+ # stop if EOS
213
+ if next_token == eos_idx:
214
+ break
215
+
216
+ return [vocab.itos[idx] for idx in curr_tokens]
217
+
218
+ else:
219
+ # beam search
220
+ beam = [(torch.tensor([[sos_idx]], device=image.device), 0.0)] # List of (sequence_tensor, score)
221
+ complete_seqs = [] # List of (sequence_tensor, score)
222
+
223
+ active_beams = beam_size # Keep track of active beams to potentially reduce k in topk
224
+
225
+ for _ in range(max_len - 1):
226
+ candidates = []
227
+
228
+ for seq_tensor, score in beam:
229
+ # check if sequence ended
230
+ if seq_tensor[0, -1].item() == eos_idx:
231
+ # Normalize score by length (optional, but good practice)
232
+ normalized_score = score / seq_tensor.shape[1]
233
+ complete_seqs.append((seq_tensor, normalized_score))
234
+ active_beams = max(1, active_beams -1) # Reduce active beams if one completed
235
+ continue
236
+
237
+ # get predictions
238
+ # Ensure seq_tensor is correctly shaped for the decoder [T, B]
239
+ # Current seq_tensor is [1, Current_Seq_Len], so .t() makes it [Current_Seq_Len, 1]
240
+ logits = self.decoder(seq_tensor.t(), img_features)
241
+ next_logits = logits[-1, 0, :] # Logits for the last token in the sequence
242
+
243
+ # top k candidates (use current active_beams for k)
244
+ if temperature <= 0:
245
+ log_probs = torch.log_softmax(next_logits, dim=0)
246
+ else:
247
+ # apply temperature
248
+ log_probs = torch.log_softmax(next_logits / temperature, dim=0)
249
+
250
+ # topk_probs are log_probs, topk_idx are indices
251
+ topk_log_probs, topk_idx = log_probs.topk(min(beam_size, active_beams))
252
+
253
+ # create new candidates
254
+ for i in range(topk_log_probs.size(0)): # Iterate up to actual number of top k found
255
+ new_seq = torch.cat([
256
+ seq_tensor,
257
+ torch.tensor([[topk_idx[i].item()]], device=image.device)
258
+ ], dim=1)
259
+ # New score is sum of log_probs
260
+ new_score = score + topk_log_probs[i].item()
261
+ candidates.append((new_seq, new_score))
262
+
263
+ if not candidates: # No more candidates to expand
264
+ break
265
+
266
+ # sort candidates by score and select top beam_size
267
+ candidates.sort(key=lambda x: x[1], reverse=True)
268
+ beam = candidates[:beam_size]
269
+
270
+ # If all remaining beams are complete, can break early
271
+ if all(b[0][0, -1].item() == eos_idx for b in beam):
272
+ for seq_tensor, score in beam:
273
+ if not any(torch.equal(seq_tensor, cs[0]) for cs in complete_seqs): # Avoid duplicates
274
+ normalized_score = score / seq_tensor.shape[1]
275
+ complete_seqs.append((seq_tensor, normalized_score))
276
+ break
277
+
278
+
279
+ # After loop, if no complete sequences, use the current beams
280
+ if not complete_seqs:
281
+ # Normalize scores for current beams as well
282
+ for seq_tensor, score in beam:
283
+ normalized_score = score / seq_tensor.shape[1]
284
+ complete_seqs.append((seq_tensor, normalized_score))
285
+
286
+ # Sort all collected sequences (completed or from final beam) by score
287
+ complete_seqs.sort(key=lambda x: x[1], reverse=True)
288
+
289
+ # Convert to list of (list of tokens, score)
290
+ results = []
291
+ for seq_tensor, score in complete_seqs:
292
+ tokens = [vocab.itos[idx] for idx in seq_tensor[0].tolist()]
293
+ results.append((tokens, score))
294
+
295
+ return results # Return all sorted beams
requirements.txt ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ altair==5.5.0
2
+ attrs==25.3.0
3
+ blinker==1.9.0
4
+ cachetools==5.5.2
5
+ certifi==2025.1.31
6
+ charset-normalizer==3.4.1
7
+ click==8.1.8
8
+ contourpy==1.3.2
9
+ cycler==0.12.1
10
+ filelock==3.18.0
11
+ fonttools==4.57.0
12
+ fsspec==2025.3.2
13
+ gitdb==4.0.12
14
+ GitPython==3.1.44
15
+ huggingface-hub==0.31.2
16
+ idna==3.10
17
+ Jinja2==3.1.6
18
+ joblib==1.4.2
19
+ jsonschema==4.23.0
20
+ jsonschema-specifications==2024.10.1
21
+ kiwisolver==1.4.8
22
+ MarkupSafe==3.0.2
23
+ matplotlib==3.10.1
24
+ mpmath==1.3.0
25
+ narwhals==1.35.0
26
+ networkx==3.4.2
27
+ nltk==3.9.1
28
+ numpy==2.2.4
29
+ packaging==24.2
30
+ pandas==2.2.3
31
+ pillow==11.2.1
32
+ protobuf==5.29.4
33
+ pyarrow==19.0.1
34
+ pycocoevalcap==1.2
35
+ pycocotools==2.0.8
36
+ pydeck==0.9.1
37
+ pyparsing==3.2.3
38
+ python-dateutil==2.9.0.post0
39
+ pytz==2025.2
40
+ PyYAML==6.0.2
41
+ referencing==0.36.2
42
+ regex==2024.11.6
43
+ requests==2.32.3
44
+ rpds-py==0.24.0
45
+ setuptools==78.1.0
46
+ six==1.17.0
47
+ smmap==5.0.2
48
+ streamlit==1.44.1
49
+ sympy==1.13.1
50
+ tenacity==9.1.2
51
+ toml==0.10.2
52
+ torch==2.6.0
53
+ torchvision==0.21.0
54
+ tornado==6.4.2
55
+ tqdm==4.67.1
56
+ typing_extensions==4.13.2
57
+ tzdata==2025.2
58
+ urllib3==2.4.0
vocab.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca4e0ed48875c2ecfc996254dfa73d31945f9572e295c9e92930dbc02906882d
3
+ size 211691