File size: 10,558 Bytes
b027f7e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2a3ef59
b027f7e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
import argparse
import torch
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from PIL import Image
import torchvision.transforms as transforms
import random
import numpy as np
from torchvision.datasets import CocoCaptions
import matplotlib.gridspec as gridspec
import os
from tqdm import tqdm

from model import EncoderCNN, CaptionTransformer, ImageCaptioningModel
from data_utils import Vocabulary, get_transforms

def main():
    parser = argparse.ArgumentParser(description="Generate Image Captions")
    
    parser.add_argument("--model_path", type=str, default="caption_model.pt",
                        help="Path to trained model (default: caption_model.pt)")
    parser.add_argument("--vocab_path", type=str, default="vocab.pkl",
                        help="Path to vocabulary file (default: vocab.pkl)")
    
    parser.add_argument("--sample_img_dir", type=str, required=True,
                        help="Path to sample images directory")
    parser.add_argument("--sample_ann_file", type=str, default="coco_sample/captions_val_sample.json",
                        help="Path to sample annotations JSON file (only used if --sample_img_dir is coco_sample/images)")
    
    parser.add_argument("--num_samples", type=int, default=6, 
                        help="Maximum number of samples to display")
    parser.add_argument("--beam_size", type=int, default=5,
                        help="Beam size for beam search")
    parser.add_argument("--temperature", type=float, default=1.0,
                        help="Sampling temperature")
    parser.add_argument("--output", type=str,
                        help="Path to save output image. If not provided, will use predictions.png")
    
    args = parser.parse_args()
    
    # Set default output filename
    if args.output is None:
        args.output = "predictions.png"
    
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    print(f"Using device: {device}")
    
    try:
        vocab = Vocabulary.load(args.vocab_path)
        print(f"Loaded vocabulary with {len(vocab)} tokens from {args.vocab_path}")
    except Exception as e:
        print(f"Error loading vocabulary: {e}")
        return
    
    encoder = EncoderCNN(embed_size=512)
    decoder = CaptionTransformer(vocab_size=len(vocab))
    model = ImageCaptioningModel(encoder, decoder)
    
    try:
        model.load_state_dict(torch.load(args.model_path, map_location=device))
        model = model.to(device)
        model.eval()
        print(f"Model loaded from {args.model_path}")
    except Exception as e:
        print(f"Error loading model: {e}")
        return
    
    model_transform = get_transforms(split="val")
    
    image_paths = []
    gt_captions_map = {}
    use_coco_dataset = args.sample_img_dir.startswith("coco_sample")

    if use_coco_dataset:
        image_root = args.sample_img_dir
        if not image_root.endswith('images'):
            image_root = os.path.join(image_root, 'images')
        
        annotation_file = args.sample_ann_file

        print(f"Attempting to load COCO dataset: images from '{image_root}', annotations from '{annotation_file}'")

        try:
            if not os.path.isdir(image_root):
                print(f"Error: Image directory not found: {image_root}")
                return
                
            sample_dataset = CocoCaptions(
                root=image_root,
                annFile=annotation_file,
                transform=None
            )
            print(f"Loaded COCO sample dataset with {len(sample_dataset)} images initially from annotation file.")

            available_img_ids = []
            if not sample_dataset.ids:
                print("No image IDs found in the annotation file.")
                return
                
            print("Checking for available image files on disk...")
            for img_id in sample_dataset.ids:
                try:
                    img_info = sample_dataset.coco.loadImgs(img_id)[0]
                    img_file_path = os.path.join(image_root, img_info['file_name'])
                    if os.path.exists(img_file_path):
                        available_img_ids.append(img_id)
                except Exception as e:
                    print(f"Warning: Could not process img_id {img_id} from annotations: {e}")
                    continue
            
            if not available_img_ids:
                 print("No valid image files found on disk corresponding to the annotations.")
                 return
            
            print(f"Found {len(available_img_ids)} available images on disk.")
            
            num_to_sample = min(args.num_samples, 6, len(available_img_ids))
            if num_to_sample == 0:
                 print("No images available to sample after checking disk.")
                 return
            
            selected_img_ids = random.sample(available_img_ids, num_to_sample)
            
            for img_id in selected_img_ids:
                img_info = sample_dataset.coco.loadImgs(img_id)[0]
                img_path = os.path.join(image_root, img_info['file_name'])
                image_paths.append(img_path)
                
                ann_ids = sample_dataset.coco.getAnnIds(imgIds=img_id)
                anns = sample_dataset.coco.loadAnns(ann_ids)
                gt_caps_for_img = [ann['caption'] for ann in anns]
                if gt_caps_for_img:
                    gt_captions_map[img_path] = random.choice(gt_caps_for_img)
                else:
                    gt_captions_map[img_path] = "N/A (no captions found for this image)"

        except FileNotFoundError:
             print(f"Error: Annotation file not found or critical issue during initial load: {annotation_file}")
             return
        except Exception as e:
            print(f"Error loading COCO sample dataset: {e}")
            return
    else:
        print(f"Loading images directly from custom directory: {args.sample_img_dir}")
        try:
            if not os.path.isdir(args.sample_img_dir):
                 print(f"Error: Directory not found: {args.sample_img_dir}")
                 return
                 
            all_files = [os.path.join(args.sample_img_dir, f) for f in os.listdir(args.sample_img_dir)]
            image_files = [f for f in all_files if os.path.isfile(f) and f.lower().endswith(('.png', '.jpg', '.jpeg'))]
            
            if not image_files:
                print(f"No image files (.png, .jpg, .jpeg) found in {args.sample_img_dir}")
                return
            
            print(f"Found {len(image_files)} images.")
            num_available = len(image_files)
            num_samples = min(args.num_samples, 6, num_available)
            image_paths = random.sample(image_files, num_samples)

        except Exception as e:
            print(f"Error listing or sampling images from {args.sample_img_dir}: {e}")
            return

    if not image_paths:
        print("No images selected for processing.")
        return
        
    num_samples = len(image_paths)

    if num_samples == 0:
        print("No images to display.")
        return
    elif num_samples <= 3:
        nrows, ncols = 1, num_samples
    else:
        nrows = 2
        ncols = (num_samples + 1) // 2

    fig = plt.figure(figsize=(5*ncols, 5*nrows))
    gs = gridspec.GridSpec(nrows, ncols, figure=fig)
    
    print(f"Generating captions for {num_samples} images...")
    
    # Wrap image_paths with tqdm for a progress bar
    for i, img_path in enumerate(tqdm(image_paths, desc="Generating Captions")):
        # Calculate grid position
        row, col = i // ncols, i % ncols
        ax = fig.add_subplot(gs[row, col])
        
        try:
            # Load original image using PIL
            original_img = Image.open(img_path).convert("RGB")
        except Exception as e:
            print(f"Error opening image {img_path}: {e}")
            ax.set_title(f"Error loading image: {os.path.basename(img_path)}", fontsize=9)
            ax.axis('off')
            continue # Skip to next image

        # Apply transform only for model input
        model_input = model_transform(original_img).unsqueeze(0).to(device)
        
        # Generate caption
        with torch.no_grad(): # Ensure no gradients are calculated
            # model.generate now returns a list of (tokens_list, score) tuples, sorted by score
            beam_results = model.generate(
                model_input, 
                vocab, 
                max_len=30, 
                beam_size=args.beam_size,
                temperature=args.temperature
            )
        
        title_parts = []
        
        # Ground Truth
        if use_coco_dataset:
            gt_caption = gt_captions_map.get(img_path, "./img")
            # Clean GT caption tokens
            gt_caption_cleaned = ' '.join([t for t in gt_caption.split() if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}]) # Assuming gt_caption is a string
            title_parts.append(f"GT: {gt_caption_cleaned}")

        if beam_results:
            # Best predicted caption
            best_pred_tokens, best_pred_score = beam_results[0]
            best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
            title_parts.append(f"Best Pred (Score: {best_pred_score:.2f}): {best_pred_caption}")

            # Other beam search results
            if len(beam_results) > 1:
                title_parts.append("--- Other Beams ---")
                for i, (tokens, score) in enumerate(beam_results[1:]):
                    if i < args.beam_size -1 : # Show up to beam_size-1 other beams
                        other_caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
                        title_parts.append(f"Beam {i+1} (Score: {score:.2f}): {other_caption}")
                    else:
                        break 
        else:
            title_parts.append("Pred: No caption generated")
            
        # Display original image and set title
        ax.imshow(np.array(original_img)) # Use np.array to ensure correct display
        ax.set_title("\n".join(title_parts), fontsize=8) # Adjust fontsize if needed
            
        ax.axis('off')
        
    plt.tight_layout(h_pad=3.0)
    
    # Save to predictions.png
    plt.savefig(args.output, bbox_inches='tight', dpi=200)
    print(f"Output saved to {args.output}")

if __name__ == "__main__":
    main()