davidyang5's picture
Update app.py
20be5ac verified
Raw History Blame Contribute Delete
5.06 kB
import gradio as gr
import torch
from PIL import Image
import torchvision.transforms as transforms
import os
from model import EncoderCNN, CaptionTransformer, ImageCaptioningModel
from data_utils import Vocabulary, get_transforms
import functools
import time
# Load model and vocabulary
def load_model(model_path="caption_model.pt", vocab_path="vocab.pkl"):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")
# Load vocabulary
vocab = Vocabulary.load(vocab_path)
print(f"Loaded vocabulary with {len(vocab)} tokens")
# Initialize and load model
encoder = EncoderCNN(embed_size=512)
decoder = CaptionTransformer(vocab_size=len(vocab))
model = ImageCaptioningModel(encoder, decoder)
model.load_state_dict(torch.load(model_path, map_location=device))
model = model.to(device)
model.eval()
print(f"Model loaded from {model_path}")
return model, vocab, device
# Initialize model, vocab and device globally
model, vocab, device = load_model()
# Get the same transforms used in generate.py
model_transform = get_transforms(split="val")
def generate_caption(image, beam_size=5):
"""
Generate caption for a single image
"""
if image is None:
return "Please upload an image"
start_time = time.time()
# Convert to PIL Image if needed
if not isinstance(image, Image.Image):
image = Image.fromarray(image)
# Convert to RGB if needed
image = image.convert("RGB")
# Apply transform for model input
model_input = model_transform(image).unsqueeze(0).to(device)
# Generate caption
with torch.no_grad():
beam_results = model.generate(
model_input,
vocab,
max_len=30,
beam_size=beam_size,
temperature=1.0
)
generation_time = time.time() - start_time
if beam_results:
# Get best prediction
best_pred_tokens, best_pred_score = beam_results[0]
best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
# Format output with score and timing
output = f"Best caption (Score: {best_pred_score:.2f}):\n{best_pred_caption}"
output += f"\n\nGeneration time: {generation_time:.2f} seconds"
# Add other beam results
if len(beam_results) > 1:
output += "\n\nOther candidates:"
for i, (tokens, score) in enumerate(beam_results[1:], 1):
caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
output += f"\n{i}. (Score: {score:.2f}): {caption}"
return output
else:
return "No caption generated"
def get_example_images():
"""Get all image files from the img directory"""
img_dir = "img"
if not os.path.exists(img_dir):
return []
# Get all image files
image_files = []
for file in os.listdir(img_dir):
if file.lower().endswith(('.png', '.jpg', '.jpeg')):
image_files.append([os.path.join(img_dir, file)])
return image_files
def create_interface():
with gr.Blocks(title="Image Captioning") as demo:
gr.Markdown("# Image Captioning Model")
gr.Markdown("Use one of the example images or upload your own")
# 1. Initialize the input component FIRST, but prevent it from showing up yet
image_input = gr.Image(type="pil", label="Upload Image", render=False)
# 2. Now setup Examples (Visual location: TOP). It works because image_input exists now.
example_images = get_example_images()
if example_images:
gr.Examples(
examples=example_images,
inputs=image_input,
label="Example Images"
)
else:
gr.Markdown("No example images found in the img directory.")
with gr.Row():
with gr.Column():
# 3. Manually render the image input HERE (Visual location: LEFT COLUMN)
image_input.render()
# Parameters
with gr.Accordion("Advanced Parameters", open=False):
beam_size = gr.Slider(minimum=1, maximum=10, value=5, step=1,
label="Beam Size")
# Generate button
generate_btn = gr.Button("Generate Caption")
with gr.Column():
# Output caption
caption_output = gr.Textbox(label="Generated Caption", lines=10)
# Set up the generate function
generate_btn.click(
fn=generate_caption,
inputs=[image_input, beam_size],
outputs=caption_output
)
return demo
if __name__ == "__main__":
demo = create_interface()
demo.queue()
demo.launch(share=True)