davidyang5 commited on
Commit
1a53bb2
·
1 Parent(s): c84f2d0

fixed quantization and added more caching

Browse files
Files changed (1) hide show
  1. app.py +49 -53
app.py CHANGED
@@ -35,24 +35,16 @@ def load_model(model_path="caption_model.pt", vocab_path="vocab.pkl"):
35
 
36
  # Load model weights
37
  model.load_state_dict(torch.load(model_path, map_location=device))
38
-
39
- # Quantize model for CPU inference
40
- if device.type == 'cpu':
41
- print("Quantizing model for CPU inference...")
42
- model = torch.quantization.quantize_dynamic(
43
- model, {torch.nn.Linear}, dtype=torch.qint8
44
- )
45
-
46
  model = model.to(device)
47
  model.eval()
48
- print(f"Model loaded and optimized from {model_path}")
49
 
 
50
  return model, vocab, device
51
 
52
  # Initialize model, vocab and device globally
53
  model, vocab, device = load_model()
54
 
55
- # Get the same transforms used in generate.py, but optimize for inference
56
  model_transform = transforms.Compose([
57
  transforms.Resize((224, 224)), # Fixed size for faster processing
58
  transforms.ToTensor(),
@@ -69,47 +61,51 @@ def generate_caption(image, beam_size=5):
69
 
70
  start_time = time.time()
71
 
72
- # Convert to PIL Image if needed
73
- if not isinstance(image, Image.Image):
74
- image = Image.fromarray(image)
75
-
76
- # Convert to RGB if needed
77
- image = image.convert("RGB")
78
-
79
- # Apply transform for model input
80
- model_input = model_transform(image).unsqueeze(0).to(device)
81
-
82
- # Generate caption with torch.inference_mode() for better performance
83
- with torch.inference_mode():
84
- beam_results = model.generate(
85
- model_input,
86
- vocab,
87
- max_len=30,
88
- beam_size=min(beam_size, 5), # Cap beam size for CPU
89
- temperature=1.0
90
- )
91
-
92
- generation_time = time.time() - start_time
93
-
94
- if beam_results:
95
- # Get best prediction
96
- best_pred_tokens, best_pred_score = beam_results[0]
97
- best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
98
 
99
- # Format output with score and timing
100
- output = f"Best caption (Score: {best_pred_score:.2f}):\n{best_pred_caption}"
101
- output += f"\n\nGeneration time: {generation_time:.2f} seconds"
102
 
103
- # Add other beam results (limited for CPU)
104
- if len(beam_results) > 1:
105
- output += "\n\nOther candidates:"
106
- for i, (tokens, score) in enumerate(beam_results[1:3], 1): # Show only top 3 results
107
- caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
108
- output += f"\n{i}. (Score: {score:.2f}): {caption}"
 
 
 
 
 
 
 
 
109
 
110
- return output
111
- else:
112
- return "No caption generated"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
113
 
114
  def get_example_images():
115
  """Get all image files from the img directory"""
@@ -128,9 +124,9 @@ def get_example_images():
128
  except:
129
  continue
130
 
131
- # Sort by file size and take only the first 8 images
132
  image_files.sort(key=lambda x: x[1])
133
- return [[path] for path, _ in image_files[:8]]
134
 
135
  def create_interface():
136
  with gr.Blocks(title="Image Captioning") as demo:
@@ -176,6 +172,6 @@ def create_interface():
176
 
177
  if __name__ == "__main__":
178
  demo = create_interface()
179
- # Set concurrency to 1 to prevent memory issues
180
- demo.queue(concurrency_count=1)
181
- demo.launch(share=True)
 
35
 
36
  # Load model weights
37
  model.load_state_dict(torch.load(model_path, map_location=device))
 
 
 
 
 
 
 
 
38
  model = model.to(device)
39
  model.eval()
 
40
 
41
+ print(f"Model loaded from {model_path}")
42
  return model, vocab, device
43
 
44
  # Initialize model, vocab and device globally
45
  model, vocab, device = load_model()
46
 
47
+ # Optimize transforms for inference
48
  model_transform = transforms.Compose([
49
  transforms.Resize((224, 224)), # Fixed size for faster processing
50
  transforms.ToTensor(),
 
61
 
62
  start_time = time.time()
63
 
64
+ try:
65
+ # Convert to PIL Image if needed
66
+ if not isinstance(image, Image.Image):
67
+ image = Image.fromarray(image)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
 
69
+ # Convert to RGB if needed
70
+ image = image.convert("RGB")
 
71
 
72
+ # Apply transform for model input
73
+ with torch.inference_mode():
74
+ model_input = model_transform(image).unsqueeze(0).to(device)
75
+
76
+ # Generate caption
77
+ beam_results = model.generate(
78
+ model_input,
79
+ vocab,
80
+ max_len=30,
81
+ beam_size=min(beam_size, 5), # Cap beam size for CPU
82
+ temperature=1.0
83
+ )
84
+
85
+ generation_time = time.time() - start_time
86
 
87
+ if beam_results:
88
+ # Get best prediction
89
+ best_pred_tokens, best_pred_score = beam_results[0]
90
+ best_pred_caption = ' '.join([t for t in best_pred_tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
91
+
92
+ # Format output with score and timing
93
+ output = f"Best caption (Score: {best_pred_score:.2f}):\n{best_pred_caption}"
94
+ output += f"\n\nGeneration time: {generation_time:.2f} seconds"
95
+
96
+ # Add other beam results (limited for CPU)
97
+ if len(beam_results) > 1:
98
+ output += "\n\nOther candidates:"
99
+ for i, (tokens, score) in enumerate(beam_results[1:3], 1): # Show only top 3 results
100
+ caption = ' '.join([t for t in tokens if t not in {"<SOS>", "<EOS>", "<PAD>", "<UNK>"}])
101
+ output += f"\n{i}. (Score: {score:.2f}): {caption}"
102
+
103
+ return output
104
+ else:
105
+ return "No caption generated"
106
+
107
+ except Exception as e:
108
+ return f"Error generating caption: {str(e)}"
109
 
110
  def get_example_images():
111
  """Get all image files from the img directory"""
 
124
  except:
125
  continue
126
 
127
+ # Sort by file size and take only the first 6 images
128
  image_files.sort(key=lambda x: x[1])
129
+ return [[path] for path, _ in image_files[:6]]
130
 
131
  def create_interface():
132
  with gr.Blocks(title="Image Captioning") as demo:
 
172
 
173
  if __name__ == "__main__":
174
  demo = create_interface()
175
+ # Configure queue with current Gradio API
176
+ demo.queue()
177
+ demo.launch(share=True, show_error=True)