File size: 5,055 Bytes
2a3ef59
 
 
 
 
 
 
c84f2d0
 
2a3ef59
ae827d9
2a3ef59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1a53bb2
ae827d9
2a3ef59
 
 
 
 
ae827d9
 
2a3ef59
 
 
ae827d9
2a3ef59
 
 
 
c84f2d0
 
ae827d9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2a3ef59
ae827d9
 
 
1a53bb2
ae827d9
 
 
 
 
 
2a3ef59
ae827d9
 
 
2a3ef59
 
 
 
 
 
 
ae827d9
2a3ef59
 
 
ae827d9
2a3ef59
ae827d9
2a3ef59
 
 
96259a0
 
2a3ef59
f943c23
 
 
 
96259a0
 
 
 
 
 
 
 
 
20be5ac
2a3ef59
 
20be5ac
 
2a3ef59
 
 
ae827d9
20be5ac
2a3ef59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1a53bb2
ae827d9
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
import gradio as gr
import torch
from PIL import Image
import torchvision.transforms as transforms
import os
from model import EncoderCNN, CaptionTransformer, ImageCaptioningModel
from data_utils import Vocabulary, get_transforms
import functools
import time

# Load model and vocabulary
def load_model(model_path="caption_model.pt", vocab_path="vocab.pkl"):
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    print(f"Using device: {device}")
    
    # Load vocabulary
    vocab = Vocabulary.load(vocab_path)
    print(f"Loaded vocabulary with {len(vocab)} tokens")
    
    # Initialize and load model
    encoder = EncoderCNN(embed_size=512)
    decoder = CaptionTransformer(vocab_size=len(vocab))
    model = ImageCaptioningModel(encoder, decoder)
    
    model.load_state_dict(torch.load(model_path, map_location=device))
    model = model.to(device)
    model.eval()
    print(f"Model loaded from {model_path}")
    
    return model, vocab, device

# Initialize model, vocab and device globally
model, vocab, device = load_model()

# Get the same transforms used in generate.py
model_transform = get_transforms(split="val")

def generate_caption(image, beam_size=5):
    """
    Generate caption for a single image
    """
    if image is None:
        return "Please upload an image"
    
    start_time = time.time()
    
    # Convert to PIL Image if needed
    if not isinstance(image, Image.Image):
        image = Image.fromarray(image)
    
    # Convert to RGB if needed
    image = image.convert("RGB")
    
    # Apply transform for model input
    model_input = model_transform(image).unsqueeze(0).to(device)
    
    # Generate caption
    with torch.no_grad():
        beam_results = model.generate(
            model_input,
            vocab,
            max_len=30,
            beam_size=beam_size,
            temperature=1.0
        )
    
    generation_time = time.time() - start_time
    
    if beam_results:
        # Get best prediction
        best_pred_tokens, best_pred_score = beam_results[0]
        best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
        
        # Format output with score and timing
        output = f"Best caption (Score: {best_pred_score:.2f}):\n{best_pred_caption}"
        output += f"\n\nGeneration time: {generation_time:.2f} seconds"
        
        # Add other beam results
        if len(beam_results) > 1:
            output += "\n\nOther candidates:"
            for i, (tokens, score) in enumerate(beam_results[1:], 1):
                caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
                output += f"\n{i}. (Score: {score:.2f}): {caption}"
        
        return output
    else:
        return "No caption generated"

def get_example_images():
    """Get all image files from the img directory"""
    img_dir = "img"
    if not os.path.exists(img_dir):
        return []
    
    # Get all image files
    image_files = []
    for file in os.listdir(img_dir):
        if file.lower().endswith(('.png', '.jpg', '.jpeg')):
            image_files.append([os.path.join(img_dir, file)])
    
    return image_files

def create_interface():
    with gr.Blocks(title="Image Captioning") as demo:
        gr.Markdown("# Image Captioning Model")
        gr.Markdown("Use one of the example images or upload your own")
        
        # 1. Initialize the input component FIRST, but prevent it from showing up yet
        image_input = gr.Image(type="pil", label="Upload Image", render=False)
        
        # 2. Now setup Examples (Visual location: TOP). It works because image_input exists now.
        example_images = get_example_images()
        if example_images:
            gr.Examples(
                examples=example_images,
                inputs=image_input,
                label="Example Images"
            )
        else:
            gr.Markdown("No example images found in the img directory.")

        with gr.Row():
            with gr.Column():
                # 3. Manually render the image input HERE (Visual location: LEFT COLUMN)
                image_input.render()
                
                # Parameters
                with gr.Accordion("Advanced Parameters", open=False):
                    beam_size = gr.Slider(minimum=1, maximum=10, value=5, step=1, 
                                            label="Beam Size")
                
                # Generate button
                generate_btn = gr.Button("Generate Caption")
            
            with gr.Column():
                # Output caption
                caption_output = gr.Textbox(label="Generated Caption", lines=10)
        
        # Set up the generate function
        generate_btn.click(
            fn=generate_caption,
            inputs=[image_input, beam_size],
            outputs=caption_output
        )
        
    return demo

if __name__ == "__main__":
    demo = create_interface()
    demo.queue()
    demo.launch(share=True)