Spaces:
Running
Running
Commit ·
b027f7e
1
Parent(s): a2d80c8
init
Browse files- __pycache__/data_utils.cpython-313.pyc +0 -0
- __pycache__/model.cpython-313.pyc +0 -0
- caption_model.pt +3 -0
- data_utils.py +176 -0
- generate.py +256 -0
- model.py +295 -0
- requirements.txt +58 -0
- vocab.pkl +3 -0
__pycache__/data_utils.cpython-313.pyc
ADDED
|
Binary file (8.73 kB). View file
|
|
|
__pycache__/model.cpython-313.pyc
ADDED
|
Binary file (11.9 kB). View file
|
|
|
caption_model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4b53627f89a4d0ef9170db0a47a30fa18dd0e79a3ce9a1859be19fbef667f142
|
| 3 |
+
size 242022730
|
data_utils.py
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import json
|
| 3 |
+
import pickle
|
| 4 |
+
import torch
|
| 5 |
+
from torch.utils.data import Dataset, DataLoader
|
| 6 |
+
import torchvision.transforms as transforms
|
| 7 |
+
from torchvision.datasets import CocoCaptions
|
| 8 |
+
from collections import Counter
|
| 9 |
+
import nltk
|
| 10 |
+
from nltk.tokenize import word_tokenize
|
| 11 |
+
import random
|
| 12 |
+
|
| 13 |
+
def ensure_nltk_resource(resource_name):
|
| 14 |
+
try:
|
| 15 |
+
nltk.data.find(f'tokenizers/{resource_name}')
|
| 16 |
+
except LookupError:
|
| 17 |
+
nltk.download(resource_name, quiet=True)
|
| 18 |
+
|
| 19 |
+
ensure_nltk_resource('punkt')
|
| 20 |
+
|
| 21 |
+
class Vocabulary:
|
| 22 |
+
"""Simple vocabulary wrapper for tokenizing and numericalizing text."""
|
| 23 |
+
def __init__(self, freq_threshold=5):
|
| 24 |
+
self.freq_threshold = freq_threshold
|
| 25 |
+
self.itos = {0: "<PAD>", 1: "<SOS>", 2: "<EOS>", 3: "<UNK>"}
|
| 26 |
+
self.stoi = {v: k for k, v in self.itos.items()}
|
| 27 |
+
|
| 28 |
+
def tokenizer(self, text):
|
| 29 |
+
"""Tokenize text to lowercase word tokens."""
|
| 30 |
+
return word_tokenize(text.lower())
|
| 31 |
+
|
| 32 |
+
def build_vocabulary(self, sentence_list):
|
| 33 |
+
"""Build vocabulary from list of sentences with frequency threshold."""
|
| 34 |
+
frequencies = Counter()
|
| 35 |
+
idx = 4 # Start after special tokens
|
| 36 |
+
|
| 37 |
+
for sentence in sentence_list:
|
| 38 |
+
tokens = self.tokenizer(sentence)
|
| 39 |
+
frequencies.update(tokens)
|
| 40 |
+
|
| 41 |
+
for word, freq in frequencies.items():
|
| 42 |
+
if freq >= self.freq_threshold:
|
| 43 |
+
self.stoi[word] = idx
|
| 44 |
+
self.itos[idx] = word
|
| 45 |
+
idx += 1
|
| 46 |
+
|
| 47 |
+
def numericalize(self, text):
|
| 48 |
+
"""Convert text to token indices."""
|
| 49 |
+
tokenized_text = self.tokenizer(text)
|
| 50 |
+
return [self.stoi.get(token, self.stoi["<UNK>"]) for token in tokenized_text]
|
| 51 |
+
|
| 52 |
+
def __len__(self):
|
| 53 |
+
return len(self.itos)
|
| 54 |
+
|
| 55 |
+
def save(self, path):
|
| 56 |
+
with open(path, "wb") as f:
|
| 57 |
+
pickle.dump(self, f)
|
| 58 |
+
|
| 59 |
+
@classmethod
|
| 60 |
+
def load(cls, path):
|
| 61 |
+
with open(path, "rb") as f:
|
| 62 |
+
return pickle.load(f)
|
| 63 |
+
|
| 64 |
+
def get_transforms(split="train"):
|
| 65 |
+
"""Get image transforms for training or validation."""
|
| 66 |
+
normalize = transforms.Normalize(
|
| 67 |
+
mean=[0.485, 0.456, 0.406],
|
| 68 |
+
std=[0.229, 0.224, 0.225]
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
if split == "train":
|
| 72 |
+
return transforms.Compose([
|
| 73 |
+
transforms.Resize((256, 256)),
|
| 74 |
+
transforms.RandomCrop(224),
|
| 75 |
+
transforms.RandomHorizontalFlip(),
|
| 76 |
+
transforms.ToTensor(),
|
| 77 |
+
normalize,
|
| 78 |
+
])
|
| 79 |
+
else:
|
| 80 |
+
return transforms.Compose([
|
| 81 |
+
transforms.Resize((224, 224)),
|
| 82 |
+
transforms.ToTensor(),
|
| 83 |
+
normalize,
|
| 84 |
+
])
|
| 85 |
+
|
| 86 |
+
class CaptionCollate:
|
| 87 |
+
"""Collate function for DataLoader that handles padding captions."""
|
| 88 |
+
def __init__(self, pad_idx, vocab):
|
| 89 |
+
self.pad_idx = pad_idx
|
| 90 |
+
self.vocab = vocab
|
| 91 |
+
|
| 92 |
+
def __call__(self, batch):
|
| 93 |
+
imgs = torch.stack([item[0] for item in batch], dim=0)
|
| 94 |
+
|
| 95 |
+
captions_batch = []
|
| 96 |
+
for item in batch:
|
| 97 |
+
caption = random.choice(item[1])
|
| 98 |
+
tokens = [self.vocab.stoi["<SOS>"]] + self.vocab.numericalize(caption) + [self.vocab.stoi["<EOS>"]]
|
| 99 |
+
captions_batch.append(torch.tensor(tokens, dtype=torch.long))
|
| 100 |
+
|
| 101 |
+
captions_padded = torch.nn.utils.rnn.pad_sequence(
|
| 102 |
+
captions_batch,
|
| 103 |
+
batch_first=True,
|
| 104 |
+
padding_value=self.pad_idx
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
return imgs, captions_padded
|
| 108 |
+
|
| 109 |
+
def build_or_load_vocab(ann_file, vocab_path="vocab.pkl", freq_threshold=5):
|
| 110 |
+
"""Build or load vocabulary from annotations file."""
|
| 111 |
+
if os.path.exists(vocab_path):
|
| 112 |
+
print(f"Loading vocabulary from {vocab_path}")
|
| 113 |
+
return Vocabulary.load(vocab_path)
|
| 114 |
+
|
| 115 |
+
print(f"Building vocabulary from {ann_file}")
|
| 116 |
+
with open(ann_file, 'r') as f:
|
| 117 |
+
annotations = json.load(f)
|
| 118 |
+
|
| 119 |
+
captions = [ann['caption'] for ann in annotations['annotations']]
|
| 120 |
+
|
| 121 |
+
vocab = Vocabulary(freq_threshold=freq_threshold)
|
| 122 |
+
vocab.build_vocabulary(captions)
|
| 123 |
+
vocab.save(vocab_path)
|
| 124 |
+
|
| 125 |
+
return vocab
|
| 126 |
+
|
| 127 |
+
def get_dataloaders(
|
| 128 |
+
train_img_folder, train_ann_file,
|
| 129 |
+
val_img_folder, val_ann_file,
|
| 130 |
+
vocab, batch_size=32, num_workers=4, limit=None
|
| 131 |
+
):
|
| 132 |
+
"""Create data loaders for training and validation."""
|
| 133 |
+
train_transform = get_transforms("train")
|
| 134 |
+
val_transform = get_transforms("val")
|
| 135 |
+
|
| 136 |
+
train_dataset = CocoCaptions(
|
| 137 |
+
root=train_img_folder,
|
| 138 |
+
annFile=train_ann_file,
|
| 139 |
+
transform=train_transform
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
val_dataset = CocoCaptions(
|
| 143 |
+
root=val_img_folder,
|
| 144 |
+
annFile=val_ann_file,
|
| 145 |
+
transform=val_transform
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
if limit:
|
| 149 |
+
train_dataset = torch.utils.data.Subset(
|
| 150 |
+
train_dataset, range(min(limit, len(train_dataset)))
|
| 151 |
+
)
|
| 152 |
+
val_dataset = torch.utils.data.Subset(
|
| 153 |
+
val_dataset, range(min(limit, len(val_dataset)))
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
collate_fn = CaptionCollate(pad_idx=vocab.stoi["<PAD>"], vocab=vocab)
|
| 157 |
+
|
| 158 |
+
train_loader = DataLoader(
|
| 159 |
+
train_dataset,
|
| 160 |
+
batch_size=batch_size,
|
| 161 |
+
shuffle=True,
|
| 162 |
+
num_workers=num_workers,
|
| 163 |
+
collate_fn=collate_fn,
|
| 164 |
+
pin_memory=True,
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
val_loader = DataLoader(
|
| 168 |
+
val_dataset,
|
| 169 |
+
batch_size=batch_size,
|
| 170 |
+
shuffle=False,
|
| 171 |
+
num_workers=num_workers,
|
| 172 |
+
collate_fn=collate_fn,
|
| 173 |
+
pin_memory=True,
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
return train_loader, val_loader
|
generate.py
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import torch
|
| 3 |
+
import matplotlib
|
| 4 |
+
matplotlib.use('Agg')
|
| 5 |
+
import matplotlib.pyplot as plt
|
| 6 |
+
from PIL import Image
|
| 7 |
+
import torchvision.transforms as transforms
|
| 8 |
+
import random
|
| 9 |
+
import numpy as np
|
| 10 |
+
from torchvision.datasets import CocoCaptions
|
| 11 |
+
import matplotlib.gridspec as gridspec
|
| 12 |
+
import os
|
| 13 |
+
from tqdm import tqdm
|
| 14 |
+
|
| 15 |
+
from model import EncoderCNN, CaptionTransformer, ImageCaptioningModel
|
| 16 |
+
from data_utils import Vocabulary, get_transforms
|
| 17 |
+
|
| 18 |
+
def main():
|
| 19 |
+
parser = argparse.ArgumentParser(description="Generate Image Captions")
|
| 20 |
+
|
| 21 |
+
parser.add_argument("--model_path", type=str, default="caption_model.pt",
|
| 22 |
+
help="Path to trained model (default: caption_model.pt)")
|
| 23 |
+
parser.add_argument("--vocab_path", type=str, default="vocab.pkl",
|
| 24 |
+
help="Path to vocabulary file (default: vocab.pkl)")
|
| 25 |
+
|
| 26 |
+
parser.add_argument("--sample_img_dir", type=str, required=True,
|
| 27 |
+
help="Path to sample images directory")
|
| 28 |
+
parser.add_argument("--sample_ann_file", type=str, default="coco_sample/captions_val_sample.json",
|
| 29 |
+
help="Path to sample annotations JSON file (only used if --sample_img_dir is coco_sample/images)")
|
| 30 |
+
|
| 31 |
+
parser.add_argument("--num_samples", type=int, default=6,
|
| 32 |
+
help="Maximum number of samples to display")
|
| 33 |
+
parser.add_argument("--beam_size", type=int, default=5,
|
| 34 |
+
help="Beam size for beam search")
|
| 35 |
+
parser.add_argument("--temperature", type=float, default=1.0,
|
| 36 |
+
help="Sampling temperature")
|
| 37 |
+
parser.add_argument("--output", type=str,
|
| 38 |
+
help="Path to save output image. If not provided, will use predictions.png")
|
| 39 |
+
|
| 40 |
+
args = parser.parse_args()
|
| 41 |
+
|
| 42 |
+
# Set default output filename
|
| 43 |
+
if args.output is None:
|
| 44 |
+
args.output = "predictions.png"
|
| 45 |
+
|
| 46 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 47 |
+
print(f"Using device: {device}")
|
| 48 |
+
|
| 49 |
+
try:
|
| 50 |
+
vocab = Vocabulary.load(args.vocab_path)
|
| 51 |
+
print(f"Loaded vocabulary with {len(vocab)} tokens from {args.vocab_path}")
|
| 52 |
+
except Exception as e:
|
| 53 |
+
print(f"Error loading vocabulary: {e}")
|
| 54 |
+
return
|
| 55 |
+
|
| 56 |
+
encoder = EncoderCNN(embed_size=512)
|
| 57 |
+
decoder = CaptionTransformer(vocab_size=len(vocab))
|
| 58 |
+
model = ImageCaptioningModel(encoder, decoder)
|
| 59 |
+
|
| 60 |
+
try:
|
| 61 |
+
model.load_state_dict(torch.load(args.model_path, map_location=device))
|
| 62 |
+
model = model.to(device)
|
| 63 |
+
model.eval()
|
| 64 |
+
print(f"Model loaded from {args.model_path}")
|
| 65 |
+
except Exception as e:
|
| 66 |
+
print(f"Error loading model: {e}")
|
| 67 |
+
return
|
| 68 |
+
|
| 69 |
+
model_transform = get_transforms(split="val")
|
| 70 |
+
|
| 71 |
+
image_paths = []
|
| 72 |
+
gt_captions_map = {}
|
| 73 |
+
use_coco_dataset = args.sample_img_dir.startswith("coco_sample")
|
| 74 |
+
|
| 75 |
+
if use_coco_dataset:
|
| 76 |
+
image_root = args.sample_img_dir
|
| 77 |
+
if not image_root.endswith('images'):
|
| 78 |
+
image_root = os.path.join(image_root, 'images')
|
| 79 |
+
|
| 80 |
+
annotation_file = args.sample_ann_file
|
| 81 |
+
|
| 82 |
+
print(f"Attempting to load COCO dataset: images from '{image_root}', annotations from '{annotation_file}'")
|
| 83 |
+
|
| 84 |
+
try:
|
| 85 |
+
if not os.path.isdir(image_root):
|
| 86 |
+
print(f"Error: Image directory not found: {image_root}")
|
| 87 |
+
return
|
| 88 |
+
|
| 89 |
+
sample_dataset = CocoCaptions(
|
| 90 |
+
root=image_root,
|
| 91 |
+
annFile=annotation_file,
|
| 92 |
+
transform=None
|
| 93 |
+
)
|
| 94 |
+
print(f"Loaded COCO sample dataset with {len(sample_dataset)} images initially from annotation file.")
|
| 95 |
+
|
| 96 |
+
available_img_ids = []
|
| 97 |
+
if not sample_dataset.ids:
|
| 98 |
+
print("No image IDs found in the annotation file.")
|
| 99 |
+
return
|
| 100 |
+
|
| 101 |
+
print("Checking for available image files on disk...")
|
| 102 |
+
for img_id in sample_dataset.ids:
|
| 103 |
+
try:
|
| 104 |
+
img_info = sample_dataset.coco.loadImgs(img_id)[0]
|
| 105 |
+
img_file_path = os.path.join(image_root, img_info['file_name'])
|
| 106 |
+
if os.path.exists(img_file_path):
|
| 107 |
+
available_img_ids.append(img_id)
|
| 108 |
+
except Exception as e:
|
| 109 |
+
print(f"Warning: Could not process img_id {img_id} from annotations: {e}")
|
| 110 |
+
continue
|
| 111 |
+
|
| 112 |
+
if not available_img_ids:
|
| 113 |
+
print("No valid image files found on disk corresponding to the annotations.")
|
| 114 |
+
return
|
| 115 |
+
|
| 116 |
+
print(f"Found {len(available_img_ids)} available images on disk.")
|
| 117 |
+
|
| 118 |
+
num_to_sample = min(args.num_samples, 6, len(available_img_ids))
|
| 119 |
+
if num_to_sample == 0:
|
| 120 |
+
print("No images available to sample after checking disk.")
|
| 121 |
+
return
|
| 122 |
+
|
| 123 |
+
selected_img_ids = random.sample(available_img_ids, num_to_sample)
|
| 124 |
+
|
| 125 |
+
for img_id in selected_img_ids:
|
| 126 |
+
img_info = sample_dataset.coco.loadImgs(img_id)[0]
|
| 127 |
+
img_path = os.path.join(image_root, img_info['file_name'])
|
| 128 |
+
image_paths.append(img_path)
|
| 129 |
+
|
| 130 |
+
ann_ids = sample_dataset.coco.getAnnIds(imgIds=img_id)
|
| 131 |
+
anns = sample_dataset.coco.loadAnns(ann_ids)
|
| 132 |
+
gt_caps_for_img = [ann['caption'] for ann in anns]
|
| 133 |
+
if gt_caps_for_img:
|
| 134 |
+
gt_captions_map[img_path] = random.choice(gt_caps_for_img)
|
| 135 |
+
else:
|
| 136 |
+
gt_captions_map[img_path] = "N/A (no captions found for this image)"
|
| 137 |
+
|
| 138 |
+
except FileNotFoundError:
|
| 139 |
+
print(f"Error: Annotation file not found or critical issue during initial load: {annotation_file}")
|
| 140 |
+
return
|
| 141 |
+
except Exception as e:
|
| 142 |
+
print(f"Error loading COCO sample dataset: {e}")
|
| 143 |
+
return
|
| 144 |
+
else:
|
| 145 |
+
print(f"Loading images directly from custom directory: {args.sample_img_dir}")
|
| 146 |
+
try:
|
| 147 |
+
if not os.path.isdir(args.sample_img_dir):
|
| 148 |
+
print(f"Error: Directory not found: {args.sample_img_dir}")
|
| 149 |
+
return
|
| 150 |
+
|
| 151 |
+
all_files = [os.path.join(args.sample_img_dir, f) for f in os.listdir(args.sample_img_dir)]
|
| 152 |
+
image_files = [f for f in all_files if os.path.isfile(f) and f.lower().endswith(('.png', '.jpg', '.jpeg'))]
|
| 153 |
+
|
| 154 |
+
if not image_files:
|
| 155 |
+
print(f"No image files (.png, .jpg, .jpeg) found in {args.sample_img_dir}")
|
| 156 |
+
return
|
| 157 |
+
|
| 158 |
+
print(f"Found {len(image_files)} images.")
|
| 159 |
+
num_available = len(image_files)
|
| 160 |
+
num_samples = min(args.num_samples, 6, num_available)
|
| 161 |
+
image_paths = random.sample(image_files, num_samples)
|
| 162 |
+
|
| 163 |
+
except Exception as e:
|
| 164 |
+
print(f"Error listing or sampling images from {args.sample_img_dir}: {e}")
|
| 165 |
+
return
|
| 166 |
+
|
| 167 |
+
if not image_paths:
|
| 168 |
+
print("No images selected for processing.")
|
| 169 |
+
return
|
| 170 |
+
|
| 171 |
+
num_samples = len(image_paths)
|
| 172 |
+
|
| 173 |
+
if num_samples == 0:
|
| 174 |
+
print("No images to display.")
|
| 175 |
+
return
|
| 176 |
+
elif num_samples <= 3:
|
| 177 |
+
nrows, ncols = 1, num_samples
|
| 178 |
+
else:
|
| 179 |
+
nrows = 2
|
| 180 |
+
ncols = (num_samples + 1) // 2
|
| 181 |
+
|
| 182 |
+
fig = plt.figure(figsize=(5*ncols, 5*nrows))
|
| 183 |
+
gs = gridspec.GridSpec(nrows, ncols, figure=fig)
|
| 184 |
+
|
| 185 |
+
print(f"Generating captions for {num_samples} images...")
|
| 186 |
+
|
| 187 |
+
# Wrap image_paths with tqdm for a progress bar
|
| 188 |
+
for i, img_path in enumerate(tqdm(image_paths, desc="Generating Captions")):
|
| 189 |
+
# Calculate grid position
|
| 190 |
+
row, col = i // ncols, i % ncols
|
| 191 |
+
ax = fig.add_subplot(gs[row, col])
|
| 192 |
+
|
| 193 |
+
try:
|
| 194 |
+
# Load original image using PIL
|
| 195 |
+
original_img = Image.open(img_path).convert("RGB")
|
| 196 |
+
except Exception as e:
|
| 197 |
+
print(f"Error opening image {img_path}: {e}")
|
| 198 |
+
ax.set_title(f"Error loading image: {os.path.basename(img_path)}", fontsize=9)
|
| 199 |
+
ax.axis('off')
|
| 200 |
+
continue # Skip to next image
|
| 201 |
+
|
| 202 |
+
# Apply transform only for model input
|
| 203 |
+
model_input = model_transform(original_img).unsqueeze(0).to(device)
|
| 204 |
+
|
| 205 |
+
# Generate caption
|
| 206 |
+
with torch.no_grad(): # Ensure no gradients are calculated
|
| 207 |
+
# model.generate now returns a list of (tokens_list, score) tuples, sorted by score
|
| 208 |
+
beam_results = model.generate(
|
| 209 |
+
model_input,
|
| 210 |
+
vocab,
|
| 211 |
+
max_len=30,
|
| 212 |
+
beam_size=args.beam_size,
|
| 213 |
+
temperature=args.temperature
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
title_parts = []
|
| 217 |
+
|
| 218 |
+
# Ground Truth
|
| 219 |
+
if use_coco_dataset:
|
| 220 |
+
gt_caption = gt_captions_map.get(img_path, "N/A")
|
| 221 |
+
# Clean GT caption tokens
|
| 222 |
+
gt_caption_cleaned = ' '.join([t for t in gt_caption.split() if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}]) # Assuming gt_caption is a string
|
| 223 |
+
title_parts.append(f"GT: {gt_caption_cleaned}")
|
| 224 |
+
|
| 225 |
+
if beam_results:
|
| 226 |
+
# Best predicted caption
|
| 227 |
+
best_pred_tokens, best_pred_score = beam_results[0]
|
| 228 |
+
best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
|
| 229 |
+
title_parts.append(f"Best Pred (Score: {best_pred_score:.2f}): {best_pred_caption}")
|
| 230 |
+
|
| 231 |
+
# Other beam search results
|
| 232 |
+
if len(beam_results) > 1:
|
| 233 |
+
title_parts.append("--- Other Beams ---")
|
| 234 |
+
for i, (tokens, score) in enumerate(beam_results[1:]):
|
| 235 |
+
if i < args.beam_size -1 : # Show up to beam_size-1 other beams
|
| 236 |
+
other_caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
|
| 237 |
+
title_parts.append(f"Beam {i+1} (Score: {score:.2f}): {other_caption}")
|
| 238 |
+
else:
|
| 239 |
+
break
|
| 240 |
+
else:
|
| 241 |
+
title_parts.append("Pred: No caption generated")
|
| 242 |
+
|
| 243 |
+
# Display original image and set title
|
| 244 |
+
ax.imshow(np.array(original_img)) # Use np.array to ensure correct display
|
| 245 |
+
ax.set_title("\n".join(title_parts), fontsize=8) # Adjust fontsize if needed
|
| 246 |
+
|
| 247 |
+
ax.axis('off')
|
| 248 |
+
|
| 249 |
+
plt.tight_layout(h_pad=3.0)
|
| 250 |
+
|
| 251 |
+
# Save to predictions.png
|
| 252 |
+
plt.savefig(args.output, bbox_inches='tight', dpi=200)
|
| 253 |
+
print(f"Output saved to {args.output}")
|
| 254 |
+
|
| 255 |
+
if __name__ == "__main__":
|
| 256 |
+
main()
|
model.py
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
import torchvision.models as models
|
| 5 |
+
|
| 6 |
+
class EncoderCNN(nn.Module):
|
| 7 |
+
"""CNN-based encoder that extracts spatial visual features from images."""
|
| 8 |
+
|
| 9 |
+
def __init__(self, embed_size=512, freeze_backbone=True, layers_to_unfreeze=0):
|
| 10 |
+
super().__init__()
|
| 11 |
+
resnet = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)
|
| 12 |
+
|
| 13 |
+
# remove final pooling and fully connected layers
|
| 14 |
+
self.cnn_backbone = nn.Sequential(*list(resnet.children())[:-2])
|
| 15 |
+
|
| 16 |
+
# add projection layer to get desired embedding size
|
| 17 |
+
self.conv = nn.Conv2d(2048, embed_size, kernel_size=1)
|
| 18 |
+
|
| 19 |
+
# freeze backbone parameters if param says
|
| 20 |
+
if freeze_backbone:
|
| 21 |
+
for param in self.cnn_backbone.parameters():
|
| 22 |
+
param.requires_grad = False
|
| 23 |
+
|
| 24 |
+
if layers_to_unfreeze > 0:
|
| 25 |
+
layers = list(self.cnn_backbone.children())
|
| 26 |
+
for i in range(min(layers_to_unfreeze, len(layers))):
|
| 27 |
+
layer_idx = len(layers) - 1 - i
|
| 28 |
+
for param in layers[layer_idx].parameters():
|
| 29 |
+
param.requires_grad = True
|
| 30 |
+
|
| 31 |
+
def forward(self, images):
|
| 32 |
+
"""
|
| 33 |
+
Extract visual features from images.
|
| 34 |
+
|
| 35 |
+
Args:
|
| 36 |
+
images: Batch of images [B, 3, H, W]
|
| 37 |
+
|
| 38 |
+
Returns:
|
| 39 |
+
features: Sequence of visual features [S, B, embed_size]
|
| 40 |
+
"""
|
| 41 |
+
# feature map from CNN backbone
|
| 42 |
+
feat_map = self.cnn_backbone(images) # [B, 2048, H', W']
|
| 43 |
+
|
| 44 |
+
# project to embedding dimension
|
| 45 |
+
emb_map = self.conv(feat_map) # [B, embed_size, H', W']
|
| 46 |
+
|
| 47 |
+
# reshape to sequence of features
|
| 48 |
+
B, D, H, W = emb_map.shape
|
| 49 |
+
features = emb_map.view(B, D, H * W).permute(2, 0, 1) # [H'*W', B, embed_size]
|
| 50 |
+
|
| 51 |
+
return features
|
| 52 |
+
|
| 53 |
+
class CaptionTransformer(nn.Module):
|
| 54 |
+
"""Transformer-based decoder for generating captions from image features."""
|
| 55 |
+
|
| 56 |
+
def __init__(
|
| 57 |
+
self,
|
| 58 |
+
vocab_size,
|
| 59 |
+
d_model=512,
|
| 60 |
+
nhead=8,
|
| 61 |
+
num_layers=6,
|
| 62 |
+
dim_feedforward=2048,
|
| 63 |
+
dropout=0.1,
|
| 64 |
+
max_len=100
|
| 65 |
+
):
|
| 66 |
+
super().__init__()
|
| 67 |
+
|
| 68 |
+
self.d_model = d_model
|
| 69 |
+
self.max_len = max_len
|
| 70 |
+
|
| 71 |
+
# token embedding layer
|
| 72 |
+
self.token_emb = nn.Embedding(vocab_size, d_model)
|
| 73 |
+
|
| 74 |
+
# positional encoding (learnable)
|
| 75 |
+
self.pos_emb = nn.Parameter(torch.zeros(max_len, 1, d_model))
|
| 76 |
+
|
| 77 |
+
# transformer decoder layers
|
| 78 |
+
decoder_layer = nn.TransformerDecoderLayer(
|
| 79 |
+
d_model=d_model,
|
| 80 |
+
nhead=nhead,
|
| 81 |
+
dim_feedforward=dim_feedforward,
|
| 82 |
+
dropout=dropout,
|
| 83 |
+
batch_first=False
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
self.transformer_decoder = nn.TransformerDecoder(
|
| 87 |
+
decoder_layer=decoder_layer,
|
| 88 |
+
num_layers=num_layers
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
# output projection to vocabulary
|
| 92 |
+
self.output_proj = nn.Linear(d_model, vocab_size)
|
| 93 |
+
|
| 94 |
+
# initialize params
|
| 95 |
+
self._init_parameters()
|
| 96 |
+
|
| 97 |
+
def _init_parameters(self):
|
| 98 |
+
"""Initialize model parameters."""
|
| 99 |
+
for p in self.parameters():
|
| 100 |
+
if p.dim() > 1:
|
| 101 |
+
nn.init.xavier_uniform_(p)
|
| 102 |
+
|
| 103 |
+
def forward(self, tgt_sequence, img_memory):
|
| 104 |
+
"""
|
| 105 |
+
Perform transformer decoding.
|
| 106 |
+
|
| 107 |
+
Args:
|
| 108 |
+
tgt_sequence: Input token sequence [T, B]
|
| 109 |
+
img_memory: Encoded image features [S, B, D]
|
| 110 |
+
|
| 111 |
+
Returns:
|
| 112 |
+
logits: Output logits [T, B, vocab_size]
|
| 113 |
+
"""
|
| 114 |
+
seq_len = tgt_sequence.shape[0]
|
| 115 |
+
|
| 116 |
+
# embed tokens and add positional encoding
|
| 117 |
+
tgt_emb = self.token_emb(tgt_sequence) * math.sqrt(self.d_model)
|
| 118 |
+
tgt_emb = tgt_emb + self.pos_emb[:seq_len]
|
| 119 |
+
|
| 120 |
+
# causal mask
|
| 121 |
+
tgt_mask = nn.Transformer.generate_square_subsequent_mask(seq_len).to(tgt_sequence.device)
|
| 122 |
+
|
| 123 |
+
# decode
|
| 124 |
+
decoded = self.transformer_decoder(
|
| 125 |
+
tgt=tgt_emb,
|
| 126 |
+
memory=img_memory,
|
| 127 |
+
tgt_mask=tgt_mask
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
# project to vocabulary
|
| 131 |
+
logits = self.output_proj(decoded)
|
| 132 |
+
|
| 133 |
+
return logits
|
| 134 |
+
|
| 135 |
+
class ImageCaptioningModel(nn.Module):
|
| 136 |
+
"""End-to-end image captioning model."""
|
| 137 |
+
|
| 138 |
+
def __init__(self, encoder, decoder):
|
| 139 |
+
super().__init__()
|
| 140 |
+
self.encoder = encoder
|
| 141 |
+
self.decoder = decoder
|
| 142 |
+
|
| 143 |
+
def forward(self, images, captions):
|
| 144 |
+
"""
|
| 145 |
+
Forward pass for training.
|
| 146 |
+
|
| 147 |
+
Args:
|
| 148 |
+
images: Batch of images [B, 3, H, W]
|
| 149 |
+
captions: Target captions [B, T]
|
| 150 |
+
|
| 151 |
+
Returns:
|
| 152 |
+
logits: Output logits [T-1, B, vocab_size]
|
| 153 |
+
"""
|
| 154 |
+
# extract image features
|
| 155 |
+
img_features = self.encoder(images) # [S, B, embed_size]
|
| 156 |
+
|
| 157 |
+
# prepare target sequence for decoder (remove last token)
|
| 158 |
+
tgt_input = captions[:, :-1].t() # [T-1, B]
|
| 159 |
+
|
| 160 |
+
# generate logits
|
| 161 |
+
output_logits = self.decoder(tgt_input, img_features)
|
| 162 |
+
|
| 163 |
+
return output_logits
|
| 164 |
+
|
| 165 |
+
def generate(self, image, vocab, max_len=20, beam_size=5, temperature=1.0):
|
| 166 |
+
"""
|
| 167 |
+
Generate caption for a single image.
|
| 168 |
+
|
| 169 |
+
Args:
|
| 170 |
+
image: Image tensor [1, 3, H, W]
|
| 171 |
+
vocab: Vocabulary object
|
| 172 |
+
max_len: Maximum caption length
|
| 173 |
+
beam_size: Size of beam for beam search (1 for greedy)
|
| 174 |
+
temperature: Sampling temperature (lower = more deterministic)
|
| 175 |
+
|
| 176 |
+
Returns:
|
| 177 |
+
caption: List of tokens
|
| 178 |
+
"""
|
| 179 |
+
self.eval()
|
| 180 |
+
|
| 181 |
+
with torch.no_grad():
|
| 182 |
+
# get image features
|
| 183 |
+
img_features = self.encoder(image) # [S, 1, embed_size]
|
| 184 |
+
|
| 185 |
+
# initialize with <SOS> token
|
| 186 |
+
sos_idx = vocab.stoi["<SOS>"]
|
| 187 |
+
eos_idx = vocab.stoi["<EOS>"]
|
| 188 |
+
|
| 189 |
+
if beam_size <= 1:
|
| 190 |
+
# greedy search
|
| 191 |
+
curr_tokens = [sos_idx]
|
| 192 |
+
|
| 193 |
+
for _ in range(max_len - 1):
|
| 194 |
+
# convert tokens to tensor
|
| 195 |
+
curr_tensor = torch.LongTensor(curr_tokens).unsqueeze(1).to(image.device)
|
| 196 |
+
|
| 197 |
+
# get next token prediction
|
| 198 |
+
logits = self.decoder(curr_tensor, img_features)
|
| 199 |
+
next_token_logits = logits[-1, 0, :]
|
| 200 |
+
|
| 201 |
+
if temperature <= 0:
|
| 202 |
+
# greedy selection
|
| 203 |
+
next_token = next_token_logits.argmax().item()
|
| 204 |
+
else:
|
| 205 |
+
# sample with temperature
|
| 206 |
+
probs = torch.softmax(next_token_logits / temperature, dim=0)
|
| 207 |
+
next_token = torch.multinomial(probs, 1).item()
|
| 208 |
+
|
| 209 |
+
# add to caption
|
| 210 |
+
curr_tokens.append(next_token)
|
| 211 |
+
|
| 212 |
+
# stop if EOS
|
| 213 |
+
if next_token == eos_idx:
|
| 214 |
+
break
|
| 215 |
+
|
| 216 |
+
return [vocab.itos[idx] for idx in curr_tokens]
|
| 217 |
+
|
| 218 |
+
else:
|
| 219 |
+
# beam search
|
| 220 |
+
beam = [(torch.tensor([[sos_idx]], device=image.device), 0.0)] # List of (sequence_tensor, score)
|
| 221 |
+
complete_seqs = [] # List of (sequence_tensor, score)
|
| 222 |
+
|
| 223 |
+
active_beams = beam_size # Keep track of active beams to potentially reduce k in topk
|
| 224 |
+
|
| 225 |
+
for _ in range(max_len - 1):
|
| 226 |
+
candidates = []
|
| 227 |
+
|
| 228 |
+
for seq_tensor, score in beam:
|
| 229 |
+
# check if sequence ended
|
| 230 |
+
if seq_tensor[0, -1].item() == eos_idx:
|
| 231 |
+
# Normalize score by length (optional, but good practice)
|
| 232 |
+
normalized_score = score / seq_tensor.shape[1]
|
| 233 |
+
complete_seqs.append((seq_tensor, normalized_score))
|
| 234 |
+
active_beams = max(1, active_beams -1) # Reduce active beams if one completed
|
| 235 |
+
continue
|
| 236 |
+
|
| 237 |
+
# get predictions
|
| 238 |
+
# Ensure seq_tensor is correctly shaped for the decoder [T, B]
|
| 239 |
+
# Current seq_tensor is [1, Current_Seq_Len], so .t() makes it [Current_Seq_Len, 1]
|
| 240 |
+
logits = self.decoder(seq_tensor.t(), img_features)
|
| 241 |
+
next_logits = logits[-1, 0, :] # Logits for the last token in the sequence
|
| 242 |
+
|
| 243 |
+
# top k candidates (use current active_beams for k)
|
| 244 |
+
if temperature <= 0:
|
| 245 |
+
log_probs = torch.log_softmax(next_logits, dim=0)
|
| 246 |
+
else:
|
| 247 |
+
# apply temperature
|
| 248 |
+
log_probs = torch.log_softmax(next_logits / temperature, dim=0)
|
| 249 |
+
|
| 250 |
+
# topk_probs are log_probs, topk_idx are indices
|
| 251 |
+
topk_log_probs, topk_idx = log_probs.topk(min(beam_size, active_beams))
|
| 252 |
+
|
| 253 |
+
# create new candidates
|
| 254 |
+
for i in range(topk_log_probs.size(0)): # Iterate up to actual number of top k found
|
| 255 |
+
new_seq = torch.cat([
|
| 256 |
+
seq_tensor,
|
| 257 |
+
torch.tensor([[topk_idx[i].item()]], device=image.device)
|
| 258 |
+
], dim=1)
|
| 259 |
+
# New score is sum of log_probs
|
| 260 |
+
new_score = score + topk_log_probs[i].item()
|
| 261 |
+
candidates.append((new_seq, new_score))
|
| 262 |
+
|
| 263 |
+
if not candidates: # No more candidates to expand
|
| 264 |
+
break
|
| 265 |
+
|
| 266 |
+
# sort candidates by score and select top beam_size
|
| 267 |
+
candidates.sort(key=lambda x: x[1], reverse=True)
|
| 268 |
+
beam = candidates[:beam_size]
|
| 269 |
+
|
| 270 |
+
# If all remaining beams are complete, can break early
|
| 271 |
+
if all(b[0][0, -1].item() == eos_idx for b in beam):
|
| 272 |
+
for seq_tensor, score in beam:
|
| 273 |
+
if not any(torch.equal(seq_tensor, cs[0]) for cs in complete_seqs): # Avoid duplicates
|
| 274 |
+
normalized_score = score / seq_tensor.shape[1]
|
| 275 |
+
complete_seqs.append((seq_tensor, normalized_score))
|
| 276 |
+
break
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
# After loop, if no complete sequences, use the current beams
|
| 280 |
+
if not complete_seqs:
|
| 281 |
+
# Normalize scores for current beams as well
|
| 282 |
+
for seq_tensor, score in beam:
|
| 283 |
+
normalized_score = score / seq_tensor.shape[1]
|
| 284 |
+
complete_seqs.append((seq_tensor, normalized_score))
|
| 285 |
+
|
| 286 |
+
# Sort all collected sequences (completed or from final beam) by score
|
| 287 |
+
complete_seqs.sort(key=lambda x: x[1], reverse=True)
|
| 288 |
+
|
| 289 |
+
# Convert to list of (list of tokens, score)
|
| 290 |
+
results = []
|
| 291 |
+
for seq_tensor, score in complete_seqs:
|
| 292 |
+
tokens = [vocab.itos[idx] for idx in seq_tensor[0].tolist()]
|
| 293 |
+
results.append((tokens, score))
|
| 294 |
+
|
| 295 |
+
return results # Return all sorted beams
|
requirements.txt
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
altair==5.5.0
|
| 2 |
+
attrs==25.3.0
|
| 3 |
+
blinker==1.9.0
|
| 4 |
+
cachetools==5.5.2
|
| 5 |
+
certifi==2025.1.31
|
| 6 |
+
charset-normalizer==3.4.1
|
| 7 |
+
click==8.1.8
|
| 8 |
+
contourpy==1.3.2
|
| 9 |
+
cycler==0.12.1
|
| 10 |
+
filelock==3.18.0
|
| 11 |
+
fonttools==4.57.0
|
| 12 |
+
fsspec==2025.3.2
|
| 13 |
+
gitdb==4.0.12
|
| 14 |
+
GitPython==3.1.44
|
| 15 |
+
huggingface-hub==0.31.2
|
| 16 |
+
idna==3.10
|
| 17 |
+
Jinja2==3.1.6
|
| 18 |
+
joblib==1.4.2
|
| 19 |
+
jsonschema==4.23.0
|
| 20 |
+
jsonschema-specifications==2024.10.1
|
| 21 |
+
kiwisolver==1.4.8
|
| 22 |
+
MarkupSafe==3.0.2
|
| 23 |
+
matplotlib==3.10.1
|
| 24 |
+
mpmath==1.3.0
|
| 25 |
+
narwhals==1.35.0
|
| 26 |
+
networkx==3.4.2
|
| 27 |
+
nltk==3.9.1
|
| 28 |
+
numpy==2.2.4
|
| 29 |
+
packaging==24.2
|
| 30 |
+
pandas==2.2.3
|
| 31 |
+
pillow==11.2.1
|
| 32 |
+
protobuf==5.29.4
|
| 33 |
+
pyarrow==19.0.1
|
| 34 |
+
pycocoevalcap==1.2
|
| 35 |
+
pycocotools==2.0.8
|
| 36 |
+
pydeck==0.9.1
|
| 37 |
+
pyparsing==3.2.3
|
| 38 |
+
python-dateutil==2.9.0.post0
|
| 39 |
+
pytz==2025.2
|
| 40 |
+
PyYAML==6.0.2
|
| 41 |
+
referencing==0.36.2
|
| 42 |
+
regex==2024.11.6
|
| 43 |
+
requests==2.32.3
|
| 44 |
+
rpds-py==0.24.0
|
| 45 |
+
setuptools==78.1.0
|
| 46 |
+
six==1.17.0
|
| 47 |
+
smmap==5.0.2
|
| 48 |
+
streamlit==1.44.1
|
| 49 |
+
sympy==1.13.1
|
| 50 |
+
tenacity==9.1.2
|
| 51 |
+
toml==0.10.2
|
| 52 |
+
torch==2.6.0
|
| 53 |
+
torchvision==0.21.0
|
| 54 |
+
tornado==6.4.2
|
| 55 |
+
tqdm==4.67.1
|
| 56 |
+
typing_extensions==4.13.2
|
| 57 |
+
tzdata==2025.2
|
| 58 |
+
urllib3==2.4.0
|
vocab.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ca4e0ed48875c2ecfc996254dfa73d31945f9572e295c9e92930dbc02906882d
|
| 3 |
+
size 211691
|