import os import json import pickle import torch from torch.utils.data import Dataset, DataLoader import torchvision.transforms as transforms from torchvision.datasets import CocoCaptions from collections import Counter import nltk from nltk.tokenize import word_tokenize import random def ensure_nltk_resource(resource_name): try: nltk.data.find(f'tokenizers/{resource_name}') except LookupError: nltk.download(resource_name, quiet=True) ensure_nltk_resource('punkt') class Vocabulary: """Simple vocabulary wrapper for tokenizing and numericalizing text.""" def __init__(self, freq_threshold=5): self.freq_threshold = freq_threshold self.itos = {0: "", 1: "", 2: "", 3: ""} self.stoi = {v: k for k, v in self.itos.items()} def tokenizer(self, text): """Tokenize text to lowercase word tokens.""" return word_tokenize(text.lower()) def build_vocabulary(self, sentence_list): """Build vocabulary from list of sentences with frequency threshold.""" frequencies = Counter() idx = 4 # Start after special tokens for sentence in sentence_list: tokens = self.tokenizer(sentence) frequencies.update(tokens) for word, freq in frequencies.items(): if freq >= self.freq_threshold: self.stoi[word] = idx self.itos[idx] = word idx += 1 def numericalize(self, text): """Convert text to token indices.""" tokenized_text = self.tokenizer(text) return [self.stoi.get(token, self.stoi[""]) for token in tokenized_text] def __len__(self): return len(self.itos) def save(self, path): with open(path, "wb") as f: pickle.dump(self, f) @classmethod def load(cls, path): with open(path, "rb") as f: return pickle.load(f) def get_transforms(split="train"): """Get image transforms for training or validation.""" normalize = transforms.Normalize( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] ) if split == "train": return transforms.Compose([ transforms.Resize((256, 256)), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ]) else: return transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), normalize, ]) class CaptionCollate: """Collate function for DataLoader that handles padding captions.""" def __init__(self, pad_idx, vocab): self.pad_idx = pad_idx self.vocab = vocab def __call__(self, batch): imgs = torch.stack([item[0] for item in batch], dim=0) captions_batch = [] for item in batch: caption = random.choice(item[1]) tokens = [self.vocab.stoi[""]] + self.vocab.numericalize(caption) + [self.vocab.stoi[""]] captions_batch.append(torch.tensor(tokens, dtype=torch.long)) captions_padded = torch.nn.utils.rnn.pad_sequence( captions_batch, batch_first=True, padding_value=self.pad_idx ) return imgs, captions_padded def build_or_load_vocab(ann_file, vocab_path="vocab.pkl", freq_threshold=5): """Build or load vocabulary from annotations file.""" if os.path.exists(vocab_path): print(f"Loading vocabulary from {vocab_path}") return Vocabulary.load(vocab_path) print(f"Building vocabulary from {ann_file}") with open(ann_file, 'r') as f: annotations = json.load(f) captions = [ann['caption'] for ann in annotations['annotations']] vocab = Vocabulary(freq_threshold=freq_threshold) vocab.build_vocabulary(captions) vocab.save(vocab_path) return vocab def get_dataloaders( train_img_folder, train_ann_file, val_img_folder, val_ann_file, vocab, batch_size=32, num_workers=4, limit=None ): """Create data loaders for training and validation.""" train_transform = get_transforms("train") val_transform = get_transforms("val") train_dataset = CocoCaptions( root=train_img_folder, annFile=train_ann_file, transform=train_transform ) val_dataset = CocoCaptions( root=val_img_folder, annFile=val_ann_file, transform=val_transform ) if limit: train_dataset = torch.utils.data.Subset( train_dataset, range(min(limit, len(train_dataset))) ) val_dataset = torch.utils.data.Subset( val_dataset, range(min(limit, len(val_dataset))) ) collate_fn = CaptionCollate(pad_idx=vocab.stoi[""], vocab=vocab) train_loader = DataLoader( train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, collate_fn=collate_fn, pin_memory=True, ) val_loader = DataLoader( val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers, collate_fn=collate_fn, pin_memory=True, ) return train_loader, val_loader