File size: 5,445 Bytes
b027f7e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
import os
import json
import pickle
import torch
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as transforms
from torchvision.datasets import CocoCaptions
from collections import Counter
import nltk
from nltk.tokenize import word_tokenize
import random

def ensure_nltk_resource(resource_name):
    try:
        nltk.data.find(f'tokenizers/{resource_name}')
    except LookupError:
        nltk.download(resource_name, quiet=True)

ensure_nltk_resource('punkt')

class Vocabulary:
    """Simple vocabulary wrapper for tokenizing and numericalizing text."""
    def __init__(self, freq_threshold=5):
        self.freq_threshold = freq_threshold
        self.itos = {0: "<PAD>", 1: "<SOS>", 2: "<EOS>", 3: "<UNK>"}
        self.stoi = {v: k for k, v in self.itos.items()}
        
    def tokenizer(self, text):
        """Tokenize text to lowercase word tokens."""
        return word_tokenize(text.lower())
        
    def build_vocabulary(self, sentence_list):
        """Build vocabulary from list of sentences with frequency threshold."""
        frequencies = Counter()
        idx = 4  # Start after special tokens
        
        for sentence in sentence_list:
            tokens = self.tokenizer(sentence)
            frequencies.update(tokens)
        
        for word, freq in frequencies.items():
            if freq >= self.freq_threshold:
                self.stoi[word] = idx
                self.itos[idx] = word
                idx += 1
                
    def numericalize(self, text):
        """Convert text to token indices."""
        tokenized_text = self.tokenizer(text)
        return [self.stoi.get(token, self.stoi["<UNK>"]) for token in tokenized_text]
        
    def __len__(self):
        return len(self.itos)
        
    def save(self, path):
        with open(path, "wb") as f:
            pickle.dump(self, f)
            
    @classmethod
    def load(cls, path):
        with open(path, "rb") as f:
            return pickle.load(f)

def get_transforms(split="train"):
    """Get image transforms for training or validation."""
    normalize = transforms.Normalize(
        mean=[0.485, 0.456, 0.406], 
        std=[0.229, 0.224, 0.225]
    )
    
    if split == "train":
        return transforms.Compose([
            transforms.Resize((256, 256)),
            transforms.RandomCrop(224),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            normalize,
        ])
    else:
        return transforms.Compose([
            transforms.Resize((224, 224)),
            transforms.ToTensor(),
            normalize,
        ])

class CaptionCollate:
    """Collate function for DataLoader that handles padding captions."""
    def __init__(self, pad_idx, vocab):
        self.pad_idx = pad_idx
        self.vocab = vocab
        
    def __call__(self, batch):
        imgs = torch.stack([item[0] for item in batch], dim=0)
        
        captions_batch = []
        for item in batch:
            caption = random.choice(item[1])
            tokens = [self.vocab.stoi["<SOS>"]] + self.vocab.numericalize(caption) + [self.vocab.stoi["<EOS>"]]
            captions_batch.append(torch.tensor(tokens, dtype=torch.long))
            
        captions_padded = torch.nn.utils.rnn.pad_sequence(
            captions_batch, 
            batch_first=True, 
            padding_value=self.pad_idx
        )
        
        return imgs, captions_padded

def build_or_load_vocab(ann_file, vocab_path="vocab.pkl", freq_threshold=5):
    """Build or load vocabulary from annotations file."""
    if os.path.exists(vocab_path):
        print(f"Loading vocabulary from {vocab_path}")
        return Vocabulary.load(vocab_path)
        
    print(f"Building vocabulary from {ann_file}")
    with open(ann_file, 'r') as f:
        annotations = json.load(f)
        
    captions = [ann['caption'] for ann in annotations['annotations']]
    
    vocab = Vocabulary(freq_threshold=freq_threshold)
    vocab.build_vocabulary(captions)
    vocab.save(vocab_path)
    
    return vocab

def get_dataloaders(
    train_img_folder, train_ann_file,
    val_img_folder, val_ann_file,
    vocab, batch_size=32, num_workers=4, limit=None
):
    """Create data loaders for training and validation."""
    train_transform = get_transforms("train")
    val_transform = get_transforms("val")
    
    train_dataset = CocoCaptions(
        root=train_img_folder,
        annFile=train_ann_file,
        transform=train_transform
    )
    
    val_dataset = CocoCaptions(
        root=val_img_folder,
        annFile=val_ann_file,
        transform=val_transform
    )
    
    if limit:
        train_dataset = torch.utils.data.Subset(
            train_dataset, range(min(limit, len(train_dataset)))
        )
        val_dataset = torch.utils.data.Subset(
            val_dataset, range(min(limit, len(val_dataset)))
        )
    
    collate_fn = CaptionCollate(pad_idx=vocab.stoi["<PAD>"], vocab=vocab)
    
    train_loader = DataLoader(
        train_dataset,
        batch_size=batch_size,
        shuffle=True,
        num_workers=num_workers,
        collate_fn=collate_fn,
        pin_memory=True,
    )
    
    val_loader = DataLoader(
        val_dataset,
        batch_size=batch_size,
        shuffle=False,
        num_workers=num_workers,
        collate_fn=collate_fn,
        pin_memory=True,
    )
    
    return train_loader, val_loader