Spaces:
Running
Running
File size: 5,055 Bytes
2a3ef59 c84f2d0 2a3ef59 ae827d9 2a3ef59 1a53bb2 ae827d9 2a3ef59 ae827d9 2a3ef59 ae827d9 2a3ef59 c84f2d0 ae827d9 2a3ef59 ae827d9 1a53bb2 ae827d9 2a3ef59 ae827d9 2a3ef59 ae827d9 2a3ef59 ae827d9 2a3ef59 ae827d9 2a3ef59 96259a0 2a3ef59 f943c23 96259a0 20be5ac 2a3ef59 20be5ac 2a3ef59 ae827d9 20be5ac 2a3ef59 1a53bb2 ae827d9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | import gradio as gr
import torch
from PIL import Image
import torchvision.transforms as transforms
import os
from model import EncoderCNN, CaptionTransformer, ImageCaptioningModel
from data_utils import Vocabulary, get_transforms
import functools
import time
# Load model and vocabulary
def load_model(model_path="caption_model.pt", vocab_path="vocab.pkl"):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")
# Load vocabulary
vocab = Vocabulary.load(vocab_path)
print(f"Loaded vocabulary with {len(vocab)} tokens")
# Initialize and load model
encoder = EncoderCNN(embed_size=512)
decoder = CaptionTransformer(vocab_size=len(vocab))
model = ImageCaptioningModel(encoder, decoder)
model.load_state_dict(torch.load(model_path, map_location=device))
model = model.to(device)
model.eval()
print(f"Model loaded from {model_path}")
return model, vocab, device
# Initialize model, vocab and device globally
model, vocab, device = load_model()
# Get the same transforms used in generate.py
model_transform = get_transforms(split="val")
def generate_caption(image, beam_size=5):
"""
Generate caption for a single image
"""
if image is None:
return "Please upload an image"
start_time = time.time()
# Convert to PIL Image if needed
if not isinstance(image, Image.Image):
image = Image.fromarray(image)
# Convert to RGB if needed
image = image.convert("RGB")
# Apply transform for model input
model_input = model_transform(image).unsqueeze(0).to(device)
# Generate caption
with torch.no_grad():
beam_results = model.generate(
model_input,
vocab,
max_len=30,
beam_size=beam_size,
temperature=1.0
)
generation_time = time.time() - start_time
if beam_results:
# Get best prediction
best_pred_tokens, best_pred_score = beam_results[0]
best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
# Format output with score and timing
output = f"Best caption (Score: {best_pred_score:.2f}):\n{best_pred_caption}"
output += f"\n\nGeneration time: {generation_time:.2f} seconds"
# Add other beam results
if len(beam_results) > 1:
output += "\n\nOther candidates:"
for i, (tokens, score) in enumerate(beam_results[1:], 1):
caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
output += f"\n{i}. (Score: {score:.2f}): {caption}"
return output
else:
return "No caption generated"
def get_example_images():
"""Get all image files from the img directory"""
img_dir = "img"
if not os.path.exists(img_dir):
return []
# Get all image files
image_files = []
for file in os.listdir(img_dir):
if file.lower().endswith(('.png', '.jpg', '.jpeg')):
image_files.append([os.path.join(img_dir, file)])
return image_files
def create_interface():
with gr.Blocks(title="Image Captioning") as demo:
gr.Markdown("# Image Captioning Model")
gr.Markdown("Use one of the example images or upload your own")
# 1. Initialize the input component FIRST, but prevent it from showing up yet
image_input = gr.Image(type="pil", label="Upload Image", render=False)
# 2. Now setup Examples (Visual location: TOP). It works because image_input exists now.
example_images = get_example_images()
if example_images:
gr.Examples(
examples=example_images,
inputs=image_input,
label="Example Images"
)
else:
gr.Markdown("No example images found in the img directory.")
with gr.Row():
with gr.Column():
# 3. Manually render the image input HERE (Visual location: LEFT COLUMN)
image_input.render()
# Parameters
with gr.Accordion("Advanced Parameters", open=False):
beam_size = gr.Slider(minimum=1, maximum=10, value=5, step=1,
label="Beam Size")
# Generate button
generate_btn = gr.Button("Generate Caption")
with gr.Column():
# Output caption
caption_output = gr.Textbox(label="Generated Caption", lines=10)
# Set up the generate function
generate_btn.click(
fn=generate_caption,
inputs=[image_input, beam_size],
outputs=caption_output
)
return demo
if __name__ == "__main__":
demo = create_interface()
demo.queue()
demo.launch(share=True) |