VisualAnnotation / data_utils.py
davidyang5's picture
init
b027f7e
Raw History Blame Contribute Delete
5.45 kB
import os
import json
import pickle
import torch
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as transforms
from torchvision.datasets import CocoCaptions
from collections import Counter
import nltk
from nltk.tokenize import word_tokenize
import random
def ensure_nltk_resource(resource_name):
try:
nltk.data.find(f'tokenizers/{resource_name}')
except LookupError:
nltk.download(resource_name, quiet=True)
ensure_nltk_resource('punkt')
class Vocabulary:
"""Simple vocabulary wrapper for tokenizing and numericalizing text."""
def __init__(self, freq_threshold=5):
self.freq_threshold = freq_threshold
self.itos = {0: "<PAD>", 1: "<SOS>", 2: "<EOS>", 3: "<UNK>"}
self.stoi = {v: k for k, v in self.itos.items()}
def tokenizer(self, text):
"""Tokenize text to lowercase word tokens."""
return word_tokenize(text.lower())
def build_vocabulary(self, sentence_list):
"""Build vocabulary from list of sentences with frequency threshold."""
frequencies = Counter()
idx = 4 # Start after special tokens
for sentence in sentence_list:
tokens = self.tokenizer(sentence)
frequencies.update(tokens)
for word, freq in frequencies.items():
if freq >= self.freq_threshold:
self.stoi[word] = idx
self.itos[idx] = word
idx += 1
def numericalize(self, text):
"""Convert text to token indices."""
tokenized_text = self.tokenizer(text)
return [self.stoi.get(token, self.stoi["<UNK>"]) for token in tokenized_text]
def __len__(self):
return len(self.itos)
def save(self, path):
with open(path, "wb") as f:
pickle.dump(self, f)
@classmethod
def load(cls, path):
with open(path, "rb") as f:
return pickle.load(f)
def get_transforms(split="train"):
"""Get image transforms for training or validation."""
normalize = transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]
)
if split == "train":
return transforms.Compose([
transforms.Resize((256, 256)),
transforms.RandomCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])
else:
return transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
normalize,
])
class CaptionCollate:
"""Collate function for DataLoader that handles padding captions."""
def __init__(self, pad_idx, vocab):
self.pad_idx = pad_idx
self.vocab = vocab
def __call__(self, batch):
imgs = torch.stack([item[0] for item in batch], dim=0)
captions_batch = []
for item in batch:
caption = random.choice(item[1])
tokens = [self.vocab.stoi["<SOS>"]] + self.vocab.numericalize(caption) + [self.vocab.stoi["<EOS>"]]
captions_batch.append(torch.tensor(tokens, dtype=torch.long))
captions_padded = torch.nn.utils.rnn.pad_sequence(
captions_batch,
batch_first=True,
padding_value=self.pad_idx
)
return imgs, captions_padded
def build_or_load_vocab(ann_file, vocab_path="vocab.pkl", freq_threshold=5):
"""Build or load vocabulary from annotations file."""
if os.path.exists(vocab_path):
print(f"Loading vocabulary from {vocab_path}")
return Vocabulary.load(vocab_path)
print(f"Building vocabulary from {ann_file}")
with open(ann_file, 'r') as f:
annotations = json.load(f)
captions = [ann['caption'] for ann in annotations['annotations']]
vocab = Vocabulary(freq_threshold=freq_threshold)
vocab.build_vocabulary(captions)
vocab.save(vocab_path)
return vocab
def get_dataloaders(
train_img_folder, train_ann_file,
val_img_folder, val_ann_file,
vocab, batch_size=32, num_workers=4, limit=None
):
"""Create data loaders for training and validation."""
train_transform = get_transforms("train")
val_transform = get_transforms("val")
train_dataset = CocoCaptions(
root=train_img_folder,
annFile=train_ann_file,
transform=train_transform
)
val_dataset = CocoCaptions(
root=val_img_folder,
annFile=val_ann_file,
transform=val_transform
)
if limit:
train_dataset = torch.utils.data.Subset(
train_dataset, range(min(limit, len(train_dataset)))
)
val_dataset = torch.utils.data.Subset(
val_dataset, range(min(limit, len(val_dataset)))
)
collate_fn = CaptionCollate(pad_idx=vocab.stoi["<PAD>"], vocab=vocab)
train_loader = DataLoader(
train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers,
collate_fn=collate_fn,
pin_memory=True,
)
val_loader = DataLoader(
val_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
collate_fn=collate_fn,
pin_memory=True,
)
return train_loader, val_loader