davidyang5 commited on
Commit
2a3ef59
·
1 Parent(s): b027f7e

added gradio app and image folder

Browse files
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ *.png filter=lfs diff=lfs merge=lfs -text
37
+ *.jpg filter=lfs diff=lfs merge=lfs -text
.gradio/certificate.pem ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ -----BEGIN CERTIFICATE-----
2
+ MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
3
+ TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
4
+ cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
5
+ WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
6
+ ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
7
+ MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
8
+ h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
9
+ 0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
10
+ A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
11
+ T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
12
+ B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
13
+ B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
14
+ KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
15
+ OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
16
+ jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
17
+ qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
18
+ rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
19
+ HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
20
+ hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
21
+ ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
22
+ 3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
23
+ NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
24
+ ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
25
+ TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
26
+ jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
27
+ oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
28
+ 4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
29
+ mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
30
+ emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
31
+ -----END CERTIFICATE-----
app.py ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import torch
3
+ from PIL import Image
4
+ import torchvision.transforms as transforms
5
+ import os
6
+ from model import EncoderCNN, CaptionTransformer, ImageCaptioningModel
7
+ from data_utils import Vocabulary, get_transforms
8
+
9
+ # Load model and vocabulary
10
+ def load_model(model_path="caption_model.pt", vocab_path="vocab.pkl"):
11
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
12
+ print(f"Using device: {device}")
13
+
14
+ # Load vocabulary
15
+ vocab = Vocabulary.load(vocab_path)
16
+ print(f"Loaded vocabulary with {len(vocab)} tokens")
17
+
18
+ # Initialize and load model
19
+ encoder = EncoderCNN(embed_size=512)
20
+ decoder = CaptionTransformer(vocab_size=len(vocab))
21
+ model = ImageCaptioningModel(encoder, decoder)
22
+
23
+ model.load_state_dict(torch.load(model_path, map_location=device))
24
+ model = model.to(device)
25
+ model.eval()
26
+ print(f"Model loaded from {model_path}")
27
+
28
+ return model, vocab, device
29
+
30
+ # Initialize model, vocab and device globally
31
+ model, vocab, device = load_model()
32
+
33
+ # Get the same transforms used in generate.py
34
+ model_transform = get_transforms(split="val")
35
+
36
+ def generate_caption(image, beam_size=5):
37
+ """
38
+ Generate caption for a single image
39
+ """
40
+ if image is None:
41
+ return "Please upload an image"
42
+
43
+ # Convert to PIL Image if needed
44
+ if not isinstance(image, Image.Image):
45
+ image = Image.fromarray(image)
46
+
47
+ # Convert to RGB if needed
48
+ image = image.convert("RGB")
49
+
50
+ # Apply transform for model input
51
+ model_input = model_transform(image).unsqueeze(0).to(device)
52
+
53
+ # Generate caption
54
+ with torch.no_grad():
55
+ beam_results = model.generate(
56
+ model_input,
57
+ vocab,
58
+ max_len=30,
59
+ beam_size=beam_size,
60
+ temperature=1.0 # Keep default temperature of 1.0
61
+ )
62
+
63
+ if beam_results:
64
+ # Get best prediction
65
+ best_pred_tokens, best_pred_score = beam_results[0]
66
+ best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
67
+
68
+ # Format output with score
69
+ output = f"Best caption (Score: {best_pred_score:.2f}):\n{best_pred_caption}"
70
+
71
+ # Add other beam results
72
+ if len(beam_results) > 1:
73
+ output += "\n\nOther candidates:"
74
+ for i, (tokens, score) in enumerate(beam_results[1:], 1):
75
+ caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
76
+ output += f"\n{i}. (Score: {score:.2f}): {caption}"
77
+
78
+ return output
79
+ else:
80
+ return "No caption generated"
81
+
82
+ def get_example_images():
83
+ """Get all image files from the img directory"""
84
+ img_dir = "img"
85
+ if not os.path.exists(img_dir):
86
+ return []
87
+
88
+ # Get all image files
89
+ image_files = []
90
+ for file in os.listdir(img_dir):
91
+ if file.lower().endswith(('.png', '.jpg', '.jpeg')):
92
+ image_files.append([os.path.join(img_dir, file)])
93
+
94
+ return image_files
95
+
96
+ def create_interface():
97
+ with gr.Blocks(title="Image Captioning") as demo:
98
+ gr.Markdown("# Image Captioning Demo")
99
+ gr.Markdown("Upload an image to generate a caption using our transformer-based model.")
100
+
101
+ with gr.Row():
102
+ with gr.Column():
103
+ # Input image
104
+ image_input = gr.Image(type="pil", label="Upload Image")
105
+
106
+ # Parameters
107
+ with gr.Accordion("Advanced Parameters", open=False):
108
+ beam_size = gr.Slider(minimum=1, maximum=10, value=5, step=1,
109
+ label="Beam Size")
110
+
111
+ # Generate button
112
+ generate_btn = gr.Button("Generate Caption")
113
+
114
+ with gr.Column():
115
+ # Output caption
116
+ caption_output = gr.Textbox(label="Generated Caption", lines=10)
117
+
118
+ # Set up the generate function
119
+ generate_btn.click(
120
+ fn=generate_caption,
121
+ inputs=[image_input, beam_size],
122
+ outputs=caption_output
123
+ )
124
+
125
+ # Example images using all images from img directory
126
+ example_images = get_example_images()
127
+ if example_images:
128
+ gr.Examples(
129
+ examples=example_images,
130
+ inputs=image_input,
131
+ label="Example Images"
132
+ )
133
+ else:
134
+ gr.Markdown("No example images found in the img directory.")
135
+
136
+ return demo
137
+
138
+ if __name__ == "__main__":
139
+ demo = create_interface()
140
+ demo.launch(share=True) # share=True creates a public URL
generate.py CHANGED
@@ -217,7 +217,7 @@ def main():
217
 
218
  # Ground Truth
219
  if use_coco_dataset:
220
- gt_caption = gt_captions_map.get(img_path, "N/A")
221
  # Clean GT caption tokens
222
  gt_caption_cleaned = ' '.join([t for t in gt_caption.split() if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}]) # Assuming gt_caption is a string
223
  title_parts.append(f"GT: {gt_caption_cleaned}")
 
217
 
218
  # Ground Truth
219
  if use_coco_dataset:
220
+ gt_caption = gt_captions_map.get(img_path, "./img")
221
  # Clean GT caption tokens
222
  gt_caption_cleaned = ' '.join([t for t in gt_caption.split() if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}]) # Assuming gt_caption is a string
223
  title_parts.append(f"GT: {gt_caption_cleaned}")
img/IMG_0747.jpg ADDED

Git LFS Details

  • SHA256: e78cecea3926709f3964184898dc29729842e9cb8c04da044029d5e846ce0f5a
  • Pointer size: 132 Bytes
  • Size of remote file: 2.21 MB
img/IMG_0749.jpg ADDED

Git LFS Details

  • SHA256: 79d193835d89017ac714465a48ce9139f4d15932a1c10871f401bcdb0ab49988
  • Pointer size: 131 Bytes
  • Size of remote file: 901 kB
img/IMG_0763.jpg ADDED

Git LFS Details

  • SHA256: 15378308f2790e712546e7a843848a6ca34915a79e168d5ad72a267db57f501c
  • Pointer size: 131 Bytes
  • Size of remote file: 728 kB
img/IMG_0811.jpg ADDED

Git LFS Details

  • SHA256: f9f1c7a8f051fedeb8dfb8e02f79c2c755af9532abdfc886f1d9ba309e14b334
  • Pointer size: 131 Bytes
  • Size of remote file: 957 kB
img/IMG_0812.jpg ADDED

Git LFS Details

  • SHA256: ec975b3ba0fb17316b1e5d4bdf9c54761ba7c5af766032b7ce746a301566d5bf
  • Pointer size: 132 Bytes
  • Size of remote file: 2.53 MB
img/IMG_0813.jpg ADDED

Git LFS Details

  • SHA256: 962528bc582657440c8c2888a7b3ca037457d7940b6c200879e0c0298bb87259
  • Pointer size: 132 Bytes
  • Size of remote file: 1.24 MB
img/Screenshot 2025-05-01 at 10.47.21 PM.png ADDED

Git LFS Details

  • SHA256: d43384d85b3fedaffd7244f19d2be9ce49c445e6b868a2c392446cf60d2712da
  • Pointer size: 131 Bytes
  • Size of remote file: 785 kB
img/Screenshot 2025-05-06 at 10.32.16 AM.png ADDED

Git LFS Details

  • SHA256: c30d586c2f6438c985d258d1b00b994909ec3c9c6c0a54692e076b2f2053e962
  • Pointer size: 132 Bytes
  • Size of remote file: 1.73 MB
img/Screenshot 2025-05-06 at 9.56.27 AM.png ADDED

Git LFS Details

  • SHA256: db07afc7f27d578021c3c3193db4a19ed5f5329d4dfc8ea36fb7903e657a7285
  • Pointer size: 132 Bytes
  • Size of remote file: 2.04 MB
img/Screenshot 2025-05-06 at 9.56.58 AM.png ADDED

Git LFS Details

  • SHA256: 1fb3bcb5a01d73169834d0757b3270177d3fe565385ff058dc208a2dfae8c073
  • Pointer size: 132 Bytes
  • Size of remote file: 2.73 MB