Spaces:
Running
Running
Download app.py from davidyang5/VisualAnnotation: direct link, hf CLI and curl.
- Browser
- Download file 5.06 kB
-
https://huggingface.co/spaces/davidyang5/VisualAnnotation/resolve/main/app.py
- Command line
-
hf download hf://spaces/davidyang5/VisualAnnotation/app.py
-
curl -L -o app.py https://huggingface.co/spaces/davidyang5/VisualAnnotation/resolve/main/app.py
5.06 kB
| import gradio as gr | |
| import torch | |
| from PIL import Image | |
| import torchvision.transforms as transforms | |
| import os | |
| from model import EncoderCNN, CaptionTransformer, ImageCaptioningModel | |
| from data_utils import Vocabulary, get_transforms | |
| import functools | |
| import time | |
| # Load model and vocabulary | |
| def load_model(model_path="caption_model.pt", vocab_path="vocab.pkl"): | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| print(f"Using device: {device}") | |
| # Load vocabulary | |
| vocab = Vocabulary.load(vocab_path) | |
| print(f"Loaded vocabulary with {len(vocab)} tokens") | |
| # Initialize and load model | |
| encoder = EncoderCNN(embed_size=512) | |
| decoder = CaptionTransformer(vocab_size=len(vocab)) | |
| model = ImageCaptioningModel(encoder, decoder) | |
| model.load_state_dict(torch.load(model_path, map_location=device)) | |
| model = model.to(device) | |
| model.eval() | |
| print(f"Model loaded from {model_path}") | |
| return model, vocab, device | |
| # Initialize model, vocab and device globally | |
| model, vocab, device = load_model() | |
| # Get the same transforms used in generate.py | |
| model_transform = get_transforms(split="val") | |
| def generate_caption(image, beam_size=5): | |
| """ | |
| Generate caption for a single image | |
| """ | |
| if image is None: | |
| return "Please upload an image" | |
| start_time = time.time() | |
| # Convert to PIL Image if needed | |
| if not isinstance(image, Image.Image): | |
| image = Image.fromarray(image) | |
| # Convert to RGB if needed | |
| image = image.convert("RGB") | |
| # Apply transform for model input | |
| model_input = model_transform(image).unsqueeze(0).to(device) | |
| # Generate caption | |
| with torch.no_grad(): | |
| beam_results = model.generate( | |
| model_input, | |
| vocab, | |
| max_len=30, | |
| beam_size=beam_size, | |
| temperature=1.0 | |
| ) | |
| generation_time = time.time() - start_time | |
| if beam_results: | |
| # Get best prediction | |
| best_pred_tokens, best_pred_score = beam_results[0] | |
| best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}]) | |
| # Format output with score and timing | |
| output = f"Best caption (Score: {best_pred_score:.2f}):\n{best_pred_caption}" | |
| output += f"\n\nGeneration time: {generation_time:.2f} seconds" | |
| # Add other beam results | |
| if len(beam_results) > 1: | |
| output += "\n\nOther candidates:" | |
| for i, (tokens, score) in enumerate(beam_results[1:], 1): | |
| caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}]) | |
| output += f"\n{i}. (Score: {score:.2f}): {caption}" | |
| return output | |
| else: | |
| return "No caption generated" | |
| def get_example_images(): | |
| """Get all image files from the img directory""" | |
| img_dir = "img" | |
| if not os.path.exists(img_dir): | |
| return [] | |
| # Get all image files | |
| image_files = [] | |
| for file in os.listdir(img_dir): | |
| if file.lower().endswith(('.png', '.jpg', '.jpeg')): | |
| image_files.append([os.path.join(img_dir, file)]) | |
| return image_files | |
| def create_interface(): | |
| with gr.Blocks(title="Image Captioning") as demo: | |
| gr.Markdown("# Image Captioning Model") | |
| gr.Markdown("Use one of the example images or upload your own") | |
| # 1. Initialize the input component FIRST, but prevent it from showing up yet | |
| image_input = gr.Image(type="pil", label="Upload Image", render=False) | |
| # 2. Now setup Examples (Visual location: TOP). It works because image_input exists now. | |
| example_images = get_example_images() | |
| if example_images: | |
| gr.Examples( | |
| examples=example_images, | |
| inputs=image_input, | |
| label="Example Images" | |
| ) | |
| else: | |
| gr.Markdown("No example images found in the img directory.") | |
| with gr.Row(): | |
| with gr.Column(): | |
| # 3. Manually render the image input HERE (Visual location: LEFT COLUMN) | |
| image_input.render() | |
| # Parameters | |
| with gr.Accordion("Advanced Parameters", open=False): | |
| beam_size = gr.Slider(minimum=1, maximum=10, value=5, step=1, | |
| label="Beam Size") | |
| # Generate button | |
| generate_btn = gr.Button("Generate Caption") | |
| with gr.Column(): | |
| # Output caption | |
| caption_output = gr.Textbox(label="Generated Caption", lines=10) | |
| # Set up the generate function | |
| generate_btn.click( | |
| fn=generate_caption, | |
| inputs=[image_input, beam_size], | |
| outputs=caption_output | |
| ) | |
| return demo | |
| if __name__ == "__main__": | |
| demo = create_interface() | |
| demo.queue() | |
| demo.launch(share=True) |