davidyang5 commited on
Commit
c84f2d0
·
1 Parent(s): 2a3ef59

added model quantization and optimized image processing

Browse files
Files changed (1) hide show
  1. app.py +60 -19
app.py CHANGED
@@ -5,8 +5,21 @@ import torchvision.transforms as transforms
5
  import os
6
  from model import EncoderCNN, CaptionTransformer, ImageCaptioningModel
7
  from data_utils import Vocabulary, get_transforms
 
 
8
 
9
- # Load model and vocabulary
 
 
 
 
 
 
 
 
 
 
 
10
  def load_model(model_path="caption_model.pt", vocab_path="vocab.pkl"):
11
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
12
  print(f"Using device: {device}")
@@ -20,26 +33,42 @@ def load_model(model_path="caption_model.pt", vocab_path="vocab.pkl"):
20
  decoder = CaptionTransformer(vocab_size=len(vocab))
21
  model = ImageCaptioningModel(encoder, decoder)
22
 
 
23
  model.load_state_dict(torch.load(model_path, map_location=device))
 
 
 
 
 
 
 
 
24
  model = model.to(device)
25
  model.eval()
26
- print(f"Model loaded from {model_path}")
27
 
28
  return model, vocab, device
29
 
30
  # Initialize model, vocab and device globally
31
  model, vocab, device = load_model()
32
 
33
- # Get the same transforms used in generate.py
34
- model_transform = get_transforms(split="val")
 
 
 
 
 
35
 
36
  def generate_caption(image, beam_size=5):
37
  """
38
- Generate caption for a single image
39
  """
40
  if image is None:
41
  return "Please upload an image"
42
 
 
 
43
  # Convert to PIL Image if needed
44
  if not isinstance(image, Image.Image):
45
  image = Image.fromarray(image)
@@ -50,28 +79,31 @@ def generate_caption(image, beam_size=5):
50
  # Apply transform for model input
51
  model_input = model_transform(image).unsqueeze(0).to(device)
52
 
53
- # Generate caption
54
- with torch.no_grad():
55
  beam_results = model.generate(
56
  model_input,
57
  vocab,
58
  max_len=30,
59
- beam_size=beam_size,
60
- temperature=1.0 # Keep default temperature of 1.0
61
  )
62
 
 
 
63
  if beam_results:
64
  # Get best prediction
65
  best_pred_tokens, best_pred_score = beam_results[0]
66
  best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
67
 
68
- # Format output with score
69
  output = f"Best caption (Score: {best_pred_score:.2f}):\n{best_pred_caption}"
 
70
 
71
- # Add other beam results
72
  if len(beam_results) > 1:
73
  output += "\n\nOther candidates:"
74
- for i, (tokens, score) in enumerate(beam_results[1:], 1):
75
  caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
76
  output += f"\n{i}. (Score: {score:.2f}): {caption}"
77
 
@@ -85,13 +117,20 @@ def get_example_images():
85
  if not os.path.exists(img_dir):
86
  return []
87
 
88
- # Get all image files
89
  image_files = []
90
  for file in os.listdir(img_dir):
91
  if file.lower().endswith(('.png', '.jpg', '.jpeg')):
92
- image_files.append([os.path.join(img_dir, file)])
 
 
 
 
 
93
 
94
- return image_files
 
 
95
 
96
  def create_interface():
97
  with gr.Blocks(title="Image Captioning") as demo:
@@ -105,8 +144,8 @@ def create_interface():
105
 
106
  # Parameters
107
  with gr.Accordion("Advanced Parameters", open=False):
108
- beam_size = gr.Slider(minimum=1, maximum=10, value=5, step=1,
109
- label="Beam Size")
110
 
111
  # Generate button
112
  generate_btn = gr.Button("Generate Caption")
@@ -128,7 +167,7 @@ def create_interface():
128
  gr.Examples(
129
  examples=example_images,
130
  inputs=image_input,
131
- label="Example Images"
132
  )
133
  else:
134
  gr.Markdown("No example images found in the img directory.")
@@ -137,4 +176,6 @@ def create_interface():
137
 
138
  if __name__ == "__main__":
139
  demo = create_interface()
140
- demo.launch(share=True) # share=True creates a public URL
 
 
 
5
  import os
6
  from model import EncoderCNN, CaptionTransformer, ImageCaptioningModel
7
  from data_utils import Vocabulary, get_transforms
8
+ import functools
9
+ import time
10
 
11
+ # Cache for storing processed images
12
+ @functools.lru_cache(maxsize=32)
13
+ def load_and_preprocess_image(image_path):
14
+ """Load and preprocess image with caching"""
15
+ try:
16
+ image = Image.open(image_path).convert("RGB")
17
+ return model_transform(image)
18
+ except Exception as e:
19
+ print(f"Error processing image {image_path}: {e}")
20
+ return None
21
+
22
+ # Load model and vocabulary with optimizations
23
  def load_model(model_path="caption_model.pt", vocab_path="vocab.pkl"):
24
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
25
  print(f"Using device: {device}")
 
33
  decoder = CaptionTransformer(vocab_size=len(vocab))
34
  model = ImageCaptioningModel(encoder, decoder)
35
 
36
+ # Load model weights
37
  model.load_state_dict(torch.load(model_path, map_location=device))
38
+
39
+ # Quantize model for CPU inference
40
+ if device.type == 'cpu':
41
+ print("Quantizing model for CPU inference...")
42
+ model = torch.quantization.quantize_dynamic(
43
+ model, {torch.nn.Linear}, dtype=torch.qint8
44
+ )
45
+
46
  model = model.to(device)
47
  model.eval()
48
+ print(f"Model loaded and optimized from {model_path}")
49
 
50
  return model, vocab, device
51
 
52
  # Initialize model, vocab and device globally
53
  model, vocab, device = load_model()
54
 
55
+ # Get the same transforms used in generate.py, but optimize for inference
56
+ model_transform = transforms.Compose([
57
+ transforms.Resize((224, 224)), # Fixed size for faster processing
58
+ transforms.ToTensor(),
59
+ transforms.Normalize(mean=[0.485, 0.456, 0.406],
60
+ std=[0.229, 0.224, 0.225])
61
+ ])
62
 
63
  def generate_caption(image, beam_size=5):
64
  """
65
+ Generate caption for a single image with optimizations
66
  """
67
  if image is None:
68
  return "Please upload an image"
69
 
70
+ start_time = time.time()
71
+
72
  # Convert to PIL Image if needed
73
  if not isinstance(image, Image.Image):
74
  image = Image.fromarray(image)
 
79
  # Apply transform for model input
80
  model_input = model_transform(image).unsqueeze(0).to(device)
81
 
82
+ # Generate caption with torch.inference_mode() for better performance
83
+ with torch.inference_mode():
84
  beam_results = model.generate(
85
  model_input,
86
  vocab,
87
  max_len=30,
88
+ beam_size=min(beam_size, 5), # Cap beam size for CPU
89
+ temperature=1.0
90
  )
91
 
92
+ generation_time = time.time() - start_time
93
+
94
  if beam_results:
95
  # Get best prediction
96
  best_pred_tokens, best_pred_score = beam_results[0]
97
  best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
98
 
99
+ # Format output with score and timing
100
  output = f"Best caption (Score: {best_pred_score:.2f}):\n{best_pred_caption}"
101
+ output += f"\n\nGeneration time: {generation_time:.2f} seconds"
102
 
103
+ # Add other beam results (limited for CPU)
104
  if len(beam_results) > 1:
105
  output += "\n\nOther candidates:"
106
+ for i, (tokens, score) in enumerate(beam_results[1:3], 1): # Show only top 3 results
107
  caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
108
  output += f"\n{i}. (Score: {score:.2f}): {caption}"
109
 
 
117
  if not os.path.exists(img_dir):
118
  return []
119
 
120
+ # Get all image files and sort by size to show smaller images first
121
  image_files = []
122
  for file in os.listdir(img_dir):
123
  if file.lower().endswith(('.png', '.jpg', '.jpeg')):
124
+ file_path = os.path.join(img_dir, file)
125
+ try:
126
+ size = os.path.getsize(file_path)
127
+ image_files.append((file_path, size))
128
+ except:
129
+ continue
130
 
131
+ # Sort by file size and take only the first 8 images
132
+ image_files.sort(key=lambda x: x[1])
133
+ return [[path] for path, _ in image_files[:8]]
134
 
135
  def create_interface():
136
  with gr.Blocks(title="Image Captioning") as demo:
 
144
 
145
  # Parameters
146
  with gr.Accordion("Advanced Parameters", open=False):
147
+ beam_size = gr.Slider(minimum=1, maximum=5, value=3, step=1,
148
+ label="Beam Size (lower is faster)")
149
 
150
  # Generate button
151
  generate_btn = gr.Button("Generate Caption")
 
167
  gr.Examples(
168
  examples=example_images,
169
  inputs=image_input,
170
+ label="Example Images (sorted by size)"
171
  )
172
  else:
173
  gr.Markdown("No example images found in the img directory.")
 
176
 
177
  if __name__ == "__main__":
178
  demo = create_interface()
179
+ # Set concurrency to 1 to prevent memory issues
180
+ demo.queue(concurrency_count=1)
181
+ demo.launch(share=True)