davidyang5 commited on
Commit
ae827d9
·
1 Parent(s): 1a53bb2

reverted changes that made it run longer

Browse files
Files changed (1) hide show
  1. app.py +50 -79
app.py CHANGED
@@ -8,18 +8,7 @@ from data_utils import Vocabulary, get_transforms
8
  import functools
9
  import time
10
 
11
- # Cache for storing processed images
12
- @functools.lru_cache(maxsize=32)
13
- def load_and_preprocess_image(image_path):
14
- """Load and preprocess image with caching"""
15
- try:
16
- image = Image.open(image_path).convert("RGB")
17
- return model_transform(image)
18
- except Exception as e:
19
- print(f"Error processing image {image_path}: {e}")
20
- return None
21
-
22
- # Load model and vocabulary with optimizations
23
  def load_model(model_path="caption_model.pt", vocab_path="vocab.pkl"):
24
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
25
  print(f"Using device: {device}")
@@ -33,79 +22,69 @@ def load_model(model_path="caption_model.pt", vocab_path="vocab.pkl"):
33
  decoder = CaptionTransformer(vocab_size=len(vocab))
34
  model = ImageCaptioningModel(encoder, decoder)
35
 
36
- # Load model weights
37
  model.load_state_dict(torch.load(model_path, map_location=device))
38
  model = model.to(device)
39
  model.eval()
40
-
41
  print(f"Model loaded from {model_path}")
 
42
  return model, vocab, device
43
 
44
  # Initialize model, vocab and device globally
45
  model, vocab, device = load_model()
46
 
47
- # Optimize transforms for inference
48
- model_transform = transforms.Compose([
49
- transforms.Resize((224, 224)), # Fixed size for faster processing
50
- transforms.ToTensor(),
51
- transforms.Normalize(mean=[0.485, 0.456, 0.406],
52
- std=[0.229, 0.224, 0.225])
53
- ])
54
 
55
  def generate_caption(image, beam_size=5):
56
  """
57
- Generate caption for a single image with optimizations
58
  """
59
  if image is None:
60
  return "Please upload an image"
61
 
62
  start_time = time.time()
63
 
64
- try:
65
- # Convert to PIL Image if needed
66
- if not isinstance(image, Image.Image):
67
- image = Image.fromarray(image)
68
-
69
- # Convert to RGB if needed
70
- image = image.convert("RGB")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
 
72
- # Apply transform for model input
73
- with torch.inference_mode():
74
- model_input = model_transform(image).unsqueeze(0).to(device)
75
-
76
- # Generate caption
77
- beam_results = model.generate(
78
- model_input,
79
- vocab,
80
- max_len=30,
81
- beam_size=min(beam_size, 5), # Cap beam size for CPU
82
- temperature=1.0
83
- )
84
 
85
- generation_time = time.time() - start_time
 
 
 
 
 
86
 
87
- if beam_results:
88
- # Get best prediction
89
- best_pred_tokens, best_pred_score = beam_results[0]
90
- best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
91
-
92
- # Format output with score and timing
93
- output = f"Best caption (Score: {best_pred_score:.2f}):\n{best_pred_caption}"
94
- output += f"\n\nGeneration time: {generation_time:.2f} seconds"
95
-
96
- # Add other beam results (limited for CPU)
97
- if len(beam_results) > 1:
98
- output += "\n\nOther candidates:"
99
- for i, (tokens, score) in enumerate(beam_results[1:3], 1): # Show only top 3 results
100
- caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
101
- output += f"\n{i}. (Score: {score:.2f}): {caption}"
102
-
103
- return output
104
- else:
105
- return "No caption generated"
106
-
107
- except Exception as e:
108
- return f"Error generating caption: {str(e)}"
109
 
110
  def get_example_images():
111
  """Get all image files from the img directory"""
@@ -113,20 +92,13 @@ def get_example_images():
113
  if not os.path.exists(img_dir):
114
  return []
115
 
116
- # Get all image files and sort by size to show smaller images first
117
  image_files = []
118
  for file in os.listdir(img_dir):
119
  if file.lower().endswith(('.png', '.jpg', '.jpeg')):
120
- file_path = os.path.join(img_dir, file)
121
- try:
122
- size = os.path.getsize(file_path)
123
- image_files.append((file_path, size))
124
- except:
125
- continue
126
 
127
- # Sort by file size and take only the first 6 images
128
- image_files.sort(key=lambda x: x[1])
129
- return [[path] for path, _ in image_files[:6]]
130
 
131
  def create_interface():
132
  with gr.Blocks(title="Image Captioning") as demo:
@@ -140,8 +112,8 @@ def create_interface():
140
 
141
  # Parameters
142
  with gr.Accordion("Advanced Parameters", open=False):
143
- beam_size = gr.Slider(minimum=1, maximum=5, value=3, step=1,
144
- label="Beam Size (lower is faster)")
145
 
146
  # Generate button
147
  generate_btn = gr.Button("Generate Caption")
@@ -163,7 +135,7 @@ def create_interface():
163
  gr.Examples(
164
  examples=example_images,
165
  inputs=image_input,
166
- label="Example Images (sorted by size)"
167
  )
168
  else:
169
  gr.Markdown("No example images found in the img directory.")
@@ -172,6 +144,5 @@ def create_interface():
172
 
173
  if __name__ == "__main__":
174
  demo = create_interface()
175
- # Configure queue with current Gradio API
176
  demo.queue()
177
- demo.launch(share=True, show_error=True)
 
8
  import functools
9
  import time
10
 
11
+ # Load model and vocabulary
 
 
 
 
 
 
 
 
 
 
 
12
  def load_model(model_path="caption_model.pt", vocab_path="vocab.pkl"):
13
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
14
  print(f"Using device: {device}")
 
22
  decoder = CaptionTransformer(vocab_size=len(vocab))
23
  model = ImageCaptioningModel(encoder, decoder)
24
 
 
25
  model.load_state_dict(torch.load(model_path, map_location=device))
26
  model = model.to(device)
27
  model.eval()
 
28
  print(f"Model loaded from {model_path}")
29
+
30
  return model, vocab, device
31
 
32
  # Initialize model, vocab and device globally
33
  model, vocab, device = load_model()
34
 
35
+ # Get the same transforms used in generate.py
36
+ model_transform = get_transforms(split="val")
 
 
 
 
 
37
 
38
  def generate_caption(image, beam_size=5):
39
  """
40
+ Generate caption for a single image
41
  """
42
  if image is None:
43
  return "Please upload an image"
44
 
45
  start_time = time.time()
46
 
47
+ # Convert to PIL Image if needed
48
+ if not isinstance(image, Image.Image):
49
+ image = Image.fromarray(image)
50
+
51
+ # Convert to RGB if needed
52
+ image = image.convert("RGB")
53
+
54
+ # Apply transform for model input
55
+ model_input = model_transform(image).unsqueeze(0).to(device)
56
+
57
+ # Generate caption
58
+ with torch.no_grad():
59
+ beam_results = model.generate(
60
+ model_input,
61
+ vocab,
62
+ max_len=30,
63
+ beam_size=beam_size,
64
+ temperature=1.0
65
+ )
66
+
67
+ generation_time = time.time() - start_time
68
+
69
+ if beam_results:
70
+ # Get best prediction
71
+ best_pred_tokens, best_pred_score = beam_results[0]
72
+ best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
73
 
74
+ # Format output with score and timing
75
+ output = f"Best caption (Score: {best_pred_score:.2f}):\n{best_pred_caption}"
76
+ output += f"\n\nGeneration time: {generation_time:.2f} seconds"
 
 
 
 
 
 
 
 
 
77
 
78
+ # Add other beam results
79
+ if len(beam_results) > 1:
80
+ output += "\n\nOther candidates:"
81
+ for i, (tokens, score) in enumerate(beam_results[1:], 1):
82
+ caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
83
+ output += f"\n{i}. (Score: {score:.2f}): {caption}"
84
 
85
+ return output
86
+ else:
87
+ return "No caption generated"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
 
89
  def get_example_images():
90
  """Get all image files from the img directory"""
 
92
  if not os.path.exists(img_dir):
93
  return []
94
 
95
+ # Get all image files
96
  image_files = []
97
  for file in os.listdir(img_dir):
98
  if file.lower().endswith(('.png', '.jpg', '.jpeg')):
99
+ image_files.append([os.path.join(img_dir, file)])
 
 
 
 
 
100
 
101
+ return image_files
 
 
102
 
103
  def create_interface():
104
  with gr.Blocks(title="Image Captioning") as demo:
 
112
 
113
  # Parameters
114
  with gr.Accordion("Advanced Parameters", open=False):
115
+ beam_size = gr.Slider(minimum=1, maximum=10, value=5, step=1,
116
+ label="Beam Size")
117
 
118
  # Generate button
119
  generate_btn = gr.Button("Generate Caption")
 
135
  gr.Examples(
136
  examples=example_images,
137
  inputs=image_input,
138
+ label="Example Images"
139
  )
140
  else:
141
  gr.Markdown("No example images found in the img directory.")
 
144
 
145
  if __name__ == "__main__":
146
  demo = create_interface()
 
147
  demo.queue()
148
+ demo.launch(share=True)