import gradio as gr import torch from PIL import Image import torchvision.transforms as transforms import os from model import EncoderCNN, CaptionTransformer, ImageCaptioningModel from data_utils import Vocabulary, get_transforms import functools import time # Load model and vocabulary def load_model(model_path="caption_model.pt", vocab_path="vocab.pkl"): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print(f"Using device: {device}") # Load vocabulary vocab = Vocabulary.load(vocab_path) print(f"Loaded vocabulary with {len(vocab)} tokens") # Initialize and load model encoder = EncoderCNN(embed_size=512) decoder = CaptionTransformer(vocab_size=len(vocab)) model = ImageCaptioningModel(encoder, decoder) model.load_state_dict(torch.load(model_path, map_location=device)) model = model.to(device) model.eval() print(f"Model loaded from {model_path}") return model, vocab, device # Initialize model, vocab and device globally model, vocab, device = load_model() # Get the same transforms used in generate.py model_transform = get_transforms(split="val") def generate_caption(image, beam_size=5): """ Generate caption for a single image """ if image is None: return "Please upload an image" start_time = time.time() # Convert to PIL Image if needed if not isinstance(image, Image.Image): image = Image.fromarray(image) # Convert to RGB if needed image = image.convert("RGB") # Apply transform for model input model_input = model_transform(image).unsqueeze(0).to(device) # Generate caption with torch.no_grad(): beam_results = model.generate( model_input, vocab, max_len=30, beam_size=beam_size, temperature=1.0 ) generation_time = time.time() - start_time if beam_results: # Get best prediction best_pred_tokens, best_pred_score = beam_results[0] best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"", "", "", ""}]) # Format output with score and timing output = f"Best caption (Score: {best_pred_score:.2f}):\n{best_pred_caption}" output += f"\n\nGeneration time: {generation_time:.2f} seconds" # Add other beam results if len(beam_results) > 1: output += "\n\nOther candidates:" for i, (tokens, score) in enumerate(beam_results[1:], 1): caption = ' '.join([t for t in tokens if t not in {"", "", "", ""}]) output += f"\n{i}. (Score: {score:.2f}): {caption}" return output else: return "No caption generated" def get_example_images(): """Get all image files from the img directory""" img_dir = "img" if not os.path.exists(img_dir): return [] # Get all image files image_files = [] for file in os.listdir(img_dir): if file.lower().endswith(('.png', '.jpg', '.jpeg')): image_files.append([os.path.join(img_dir, file)]) return image_files def create_interface(): with gr.Blocks(title="Image Captioning") as demo: gr.Markdown("# Image Captioning Model") gr.Markdown("Use one of the example images or upload your own") # 1. Initialize the input component FIRST, but prevent it from showing up yet image_input = gr.Image(type="pil", label="Upload Image", render=False) # 2. Now setup Examples (Visual location: TOP). It works because image_input exists now. example_images = get_example_images() if example_images: gr.Examples( examples=example_images, inputs=image_input, label="Example Images" ) else: gr.Markdown("No example images found in the img directory.") with gr.Row(): with gr.Column(): # 3. Manually render the image input HERE (Visual location: LEFT COLUMN) image_input.render() # Parameters with gr.Accordion("Advanced Parameters", open=False): beam_size = gr.Slider(minimum=1, maximum=10, value=5, step=1, label="Beam Size") # Generate button generate_btn = gr.Button("Generate Caption") with gr.Column(): # Output caption caption_output = gr.Textbox(label="Generated Caption", lines=10) # Set up the generate function generate_btn.click( fn=generate_caption, inputs=[image_input, beam_size], outputs=caption_output ) return demo if __name__ == "__main__": demo = create_interface() demo.queue() demo.launch(share=True)