Workpam commited on
Commit
b70923e
·
verified ·
1 Parent(s): 0712911

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +100 -294
app.py CHANGED
@@ -1,348 +1,154 @@
1
  import torch
2
  from PIL import Image
3
  import torchvision.transforms as T
4
- from torchvision.models import detection
5
- from transformers import Blip2Processor, Blip2ForConditionalGeneration
6
  import gradio as gr
7
- import logging
8
- from typing import List, Optional
9
- import gc
10
- from functools import lru_cache
11
 
12
- # Setup logging
13
- logging.basicConfig(level=logging.INFO)
14
- logger = logging.getLogger(__name__)
15
-
16
- # Device setup with memory optimization
17
  device = "cuda" if torch.cuda.is_available() else "cpu"
 
 
 
 
 
 
 
 
 
 
 
 
18
  if device == "cuda":
19
- torch.cuda.empty_cache()
20
- # Enable memory efficient attention if available
21
- torch.backends.cuda.enable_math_sdp(True)
22
 
23
- # COCO class labels
24
- COCO_LABELS = [
25
- '__background__', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat',
26
- 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep',
27
- 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
28
- 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard',
29
- 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich',
30
- 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed',
31
- 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven',
32
- 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'
33
- ]
34
 
35
- class OptimizedImageCaptioner:
36
- def __init__(self):
37
- self.device = device
38
- self.detector = None
39
- self.processor = None
40
- self.model = None
41
- self._load_models()
42
 
43
- def _load_models(self):
44
- """Load models with error handling and optimization"""
45
- try:
46
- # Load object detector with optimizations
47
- logger.info("Loading object detection model...")
48
- self.detector = detection.fasterrcnn_resnet50_fpn(weights='DEFAULT')
49
- self.detector.eval()
50
- self.detector.to(self.device)
51
-
52
- # Optimize detector for inference
53
- if hasattr(torch.jit, 'optimize_for_inference'):
54
- self.detector = torch.jit.optimize_for_inference(torch.jit.script(self.detector))
55
-
56
- # Load BLIP-2 with memory optimizations
57
- logger.info("Loading BLIP-2 model...")
58
- model_name = "Salesforce/blip2-flan-t5-xl"
59
-
60
- # Use appropriate dtype based on device
61
- dtype = torch.float16 if self.device == "cuda" else torch.float32
62
-
63
- self.processor = Blip2Processor.from_pretrained(
64
- model_name,
65
- torch_dtype=dtype,
66
- low_cpu_mem_usage=True
67
- )
68
-
69
- self.model = Blip2ForConditionalGeneration.from_pretrained(
70
- model_name,
71
- torch_dtype=dtype,
72
- low_cpu_mem_usage=True,
73
- device_map="auto" if self.device == "cuda" else None
74
  )
75
-
76
- if self.device != "cuda":
77
- self.model.to(self.device)
78
-
79
- self.model.eval()
80
-
81
- # Enable gradient checkpointing for memory efficiency
82
- if hasattr(self.model, 'gradient_checkpointing_enable'):
83
- self.model.gradient_checkpointing_enable()
84
-
85
- logger.info("Models loaded successfully!")
86
-
87
- except Exception as e:
88
- logger.error(f"Error loading models: {e}")
89
- raise
90
-
91
- @lru_cache(maxsize=32)
92
- def _get_transform(self, size: int = 384):
93
- """Cached transform to avoid recreation"""
94
- return T.Compose([
95
- T.Resize((size, size), interpolation=T.InterpolationMode.BILINEAR),
96
- T.ToTensor(),
97
- T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
98
- ])
99
-
100
- def detect_objects(self, image_tensor: torch.Tensor, confidence_threshold: float = 0.7) -> List[str]:
101
- """Optimized object detection with better filtering"""
102
- try:
103
- with torch.no_grad():
104
- predictions = self.detector(image_tensor)[0]
105
-
106
- # Filter by confidence and remove duplicates
107
- detected_objects = []
108
- seen_labels = set()
109
-
110
- for label_idx, score in zip(predictions["labels"].cpu().tolist(),
111
- predictions["scores"].cpu().tolist()):
112
- if score > confidence_threshold and label_idx < len(COCO_LABELS):
113
- label = COCO_LABELS[label_idx]
114
- if label != '__background__' and label not in seen_labels:
115
- detected_objects.append((label, score))
116
- seen_labels.add(label)
117
-
118
- # Sort by confidence and return top objects
119
- detected_objects.sort(key=lambda x: x[1], reverse=True)
120
- return [obj[0] for obj in detected_objects[:10]] # Limit to top 10
121
-
122
- except Exception as e:
123
- logger.warning(f"Object detection failed: {e}")
124
- return []
125
-
126
- def generate_caption(self, image: Image.Image, detected_objects: List[str],
127
- max_new_tokens: int = 32) -> str:
128
- """Generate caption with improved prompting"""
129
- try:
130
- # Create context-aware prompt
131
- if detected_objects:
132
- objects_str = ", ".join(detected_objects[:5]) # Limit for token efficiency
133
- prompt = f"Objects detected: {objects_str}. Describe this image with specific details about colors, materials, actions, and setting:"
134
- else:
135
- prompt = "Describe this image with specific details about what you see, including colors, materials, actions, and setting:"
136
-
137
- # Process inputs
138
- inputs = self.processor(
139
- images=image,
140
- text=prompt,
141
- return_tensors="pt",
142
- padding=True,
143
- truncation=True,
144
- max_length=512
145
- ).to(self.device)
146
-
147
- # Generate with optimized parameters
148
- with torch.no_grad():
149
- outputs = self.model.generate(
150
- **inputs,
151
- max_new_tokens=max_new_tokens,
152
- num_beams=3,
153
- do_sample=False, # Deterministic for consistency
154
- no_repeat_ngram_size=3,
155
- early_stopping=True,
156
- pad_token_id=self.processor.tokenizer.pad_token_id,
157
- eos_token_id=self.processor.tokenizer.eos_token_id,
158
- length_penalty=1.0,
159
- repetition_penalty=1.1
160
- )
161
-
162
- # Decode and clean up
163
- caption = self.processor.decode(outputs[0], skip_special_tokens=True)
164
-
165
- # Remove the prompt from the output if it's included
166
- if prompt in caption:
167
- caption = caption.replace(prompt, "").strip()
168
-
169
- return caption.strip() if caption.strip() else "Unable to generate description."
170
-
171
- except Exception as e:
172
- logger.error(f"Caption generation failed: {e}")
173
- return f"Error generating caption: {str(e)}"
174
-
175
- def process_image(self, image_path: str, max_new_tokens: int = 32) -> str:
176
- """Main processing pipeline with error handling"""
177
- try:
178
- # Load and validate image
179
- image = Image.open(image_path).convert("RGB")
180
-
181
- # Check image size and resize if too large
182
- max_size = 1024
183
- if max(image.size) > max_size:
184
- image.thumbnail((max_size, max_size), Image.Resampling.LANCZOS)
185
-
186
- # Prepare image for object detection
187
- transform = self._get_transform()
188
- image_tensor = transform(image).unsqueeze(0).to(self.device)
189
-
190
- # Detect objects
191
- detected_objects = self.detect_objects(image_tensor)
192
-
193
- # Generate caption
194
- caption = self.generate_caption(image, detected_objects, max_new_tokens)
195
-
196
- # Clean up GPU memory
197
- if self.device == "cuda":
198
- del image_tensor
199
- torch.cuda.empty_cache()
200
-
201
- return caption.capitalize() if caption else "No description available."
202
-
203
- except Exception as e:
204
- logger.error(f"Error processing image {image_path}: {e}")
205
- return f"Error processing image: {str(e)}"
206
-
207
- # Global captioner instance
208
- captioner = OptimizedImageCaptioner()
209
 
210
- def process_multiple_images(files: Optional[List], max_new_tokens: int) -> str:
211
- """Process multiple images with progress tracking"""
212
  if not files:
213
  return "No images uploaded."
214
 
 
215
  results = []
216
- total_files = len(files)
217
 
 
218
  for i, file in enumerate(files):
219
- try:
220
- logger.info(f"Processing image {i+1}/{total_files}: {file.name}")
221
- caption = captioner.process_image(file.name, max_new_tokens=int(max_new_tokens))
222
- results.append(f"Image {i+1}: {caption}")
223
-
224
- # Memory cleanup between images
225
- if captioner.device == "cuda" and i % 3 == 0: # Every 3 images
226
- torch.cuda.empty_cache()
227
- gc.collect()
228
-
229
- except Exception as e:
230
- logger.error(f"Failed to process {file.name}: {e}")
231
- results.append(f"Image {i+1}: Error - {str(e)}")
232
 
 
233
  return "\n\n".join(results)
234
 
235
- def preview_images(files):
236
- """Preview uploaded images with error handling"""
237
  if not files:
238
  return []
239
 
240
  previews = []
241
- for file in files[:20]: # Limit previews to prevent UI overload
242
  try:
243
  img = Image.open(file.name).convert("RGB")
244
- # Thumbnail for preview
245
- img.thumbnail((300, 300), Image.Resampling.LANCZOS)
246
  previews.append(img)
247
- except Exception as e:
248
- logger.warning(f"Failed to preview {file.name}: {e}")
249
 
250
  return previews
251
 
252
- # Gradio Interface with improved design
253
  with gr.Blocks(
254
- title="Advanced Image Description Generator",
255
- theme=gr.themes.Soft(),
256
- css="""
257
- .gradio-container {
258
- max-width: 1200px !important;
259
- }
260
- """
261
  ) as demo:
262
 
263
  gr.Markdown("""
264
- # 🖼️ Advanced Image Description Generator
265
 
266
- Generate detailed, accurate descriptions of your images using state-of-the-art AI models.
267
- - **Object Detection**: Identifies objects in images for better context
268
- - **Advanced Captioning**: Uses BLIP-2 for detailed, natural descriptions
269
- - **Batch Processing**: Handle multiple images at once
270
  """)
271
 
272
  with gr.Row():
273
- with gr.Column(scale=1):
274
  file_input = gr.File(
275
- label="📁 Upload Images",
276
  file_count="multiple",
277
- file_types=["image"],
278
- height=200
279
  )
280
 
281
- with gr.Row():
282
- max_tokens_slider = gr.Slider(
283
- minimum=16,
284
- maximum=100,
285
- step=8,
286
- value=48,
287
- label="🎯 Description Length (tokens)",
288
- info="Higher = more detailed descriptions"
289
- )
290
 
291
  with gr.Row():
292
- generate_btn = gr.Button(
293
- "✨ Generate Descriptions",
294
- variant="primary",
295
- size="lg"
296
- )
297
- clear_btn = gr.Button(
298
- "��️ Clear All",
299
- variant="secondary"
300
- )
301
 
302
- with gr.Column(scale=2):
303
  gallery = gr.Gallery(
304
- label="🖼️ Image Preview",
305
  columns=3,
306
- rows=2,
307
- height=400,
308
- object_fit="contain"
309
  )
310
 
311
- output_textbox = gr.Textbox(
312
- label="📝 Generated Descriptions",
313
- lines=8,
314
- max_lines=20,
315
- show_copy_button=True,
316
- placeholder="Descriptions will appear here after processing..."
317
- )
318
-
319
- # Event handlers
320
- file_input.change(
321
- fn=preview_images,
322
- inputs=file_input,
323
- outputs=gallery,
324
- show_progress=True
325
  )
326
 
327
- generate_btn.click(
328
- fn=process_multiple_images,
329
- inputs=[file_input, max_tokens_slider],
330
- outputs=output_textbox,
331
- show_progress=True
332
- )
333
-
334
- clear_btn.click(
335
- fn=lambda: (None, [], 48, ""),
336
- outputs=[file_input, gallery, max_tokens_slider, output_textbox]
337
- )
338
 
339
- # Launch configuration
340
  if __name__ == "__main__":
341
- demo.launch(
342
- server_name="0.0.0.0",
343
- server_port=7860,
344
- share=False,
345
- debug=False,
346
- show_error=True,
347
- inbrowser=True
348
- )
 
1
  import torch
2
  from PIL import Image
3
  import torchvision.transforms as T
4
+ from transformers import BlipProcessor, BlipForConditionalGeneration
 
5
  import gradio as gr
6
+ import asyncio
7
+ from concurrent.futures import ThreadPoolExecutor
8
+ import time
 
9
 
10
+ # Device setup
 
 
 
 
11
  device = "cuda" if torch.cuda.is_available() else "cpu"
12
+ print(f"Using device: {device}")
13
+
14
+ # Use BLIP-1 instead of BLIP-2 for much faster inference
15
+ model_name = "Salesforce/blip-image-captioning-base"
16
+ processor = BlipProcessor.from_pretrained(model_name)
17
+ model = BlipForConditionalGeneration.from_pretrained(model_name)
18
+
19
+ # Move to device and optimize
20
+ model.to(device)
21
+ model.eval()
22
+
23
+ # Enable inference optimizations
24
  if device == "cuda":
25
+ model = model.half() # Use half precision for speed
26
+ torch.backends.cudnn.benchmark = True
 
27
 
28
+ # Fast preprocessing - smaller size for speed
29
+ transform = T.Compose([
30
+ T.Resize((224, 224)), # Smaller size = faster processing
31
+ T.ToTensor(),
32
+ ])
 
 
 
 
 
 
33
 
34
+ def generate_fast_caption(image_path, max_length=20):
35
+ """Ultra-fast caption generation"""
36
+ try:
37
+ # Load image
38
+ image = Image.open(image_path).convert("RGB")
 
 
39
 
40
+ # Resize if too large (speed optimization)
41
+ if max(image.size) > 512:
42
+ image.thumbnail((512, 512), Image.Resampling.LANCZOS)
43
+
44
+ # Simple prompt for basic captioning
45
+ inputs = processor(image, return_tensors="pt").to(device)
46
+
47
+ # Fast generation with minimal beams
48
+ with torch.no_grad():
49
+ outputs = model.generate(
50
+ **inputs,
51
+ max_length=max_length,
52
+ num_beams=1, # Single beam for speed
53
+ do_sample=False,
54
+ early_stopping=True,
55
+ pad_token_id=processor.tokenizer.pad_token_id
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
  )
57
+
58
+ caption = processor.decode(outputs[0], skip_special_tokens=True)
59
+ return caption.strip()
60
+
61
+ except Exception as e:
62
+ return f"Error: {str(e)}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
 
64
+ def process_images_fast(files, max_length):
65
+ """Process multiple images quickly"""
66
  if not files:
67
  return "No images uploaded."
68
 
69
+ start_time = time.time()
70
  results = []
 
71
 
72
+ # Process images sequentially for stability
73
  for i, file in enumerate(files):
74
+ caption = generate_fast_caption(file.name, max_length=int(max_length))
75
+ results.append(f"Image {i+1}: {caption}")
76
+
77
+ # Quick memory cleanup for CUDA
78
+ if device == "cuda" and i % 5 == 0:
79
+ torch.cuda.empty_cache()
80
+
81
+ end_time = time.time()
82
+ processing_time = end_time - start_time
 
 
 
 
83
 
84
+ results.append(f"\n⚡ Processed {len(files)} images in {processing_time:.2f} seconds")
85
  return "\n\n".join(results)
86
 
87
+ def preview_images_fast(files):
88
+ """Quick image preview"""
89
  if not files:
90
  return []
91
 
92
  previews = []
93
+ for file in files[:12]: # Limit for speed
94
  try:
95
  img = Image.open(file.name).convert("RGB")
96
+ img.thumbnail((200, 200), Image.Resampling.LANCZOS)
 
97
  previews.append(img)
98
+ except:
99
+ continue
100
 
101
  return previews
102
 
103
+ # Streamlined Gradio Interface
104
  with gr.Blocks(
105
+ title="⚡ Fast Image Captioning",
106
+ theme=gr.themes.Default()
 
 
 
 
 
107
  ) as demo:
108
 
109
  gr.Markdown("""
110
+ # ⚡ Ultra-Fast Image Captioning
111
 
112
+ Quick and simple image descriptions optimized for speed!
 
 
 
113
  """)
114
 
115
  with gr.Row():
116
+ with gr.Column():
117
  file_input = gr.File(
118
+ label="Upload Images",
119
  file_count="multiple",
120
+ file_types=["image"]
 
121
  )
122
 
123
+ max_length_slider = gr.Slider(
124
+ minimum=10,
125
+ maximum=30,
126
+ step=5,
127
+ value=20,
128
+ label="Caption Length"
129
+ )
 
 
130
 
131
  with gr.Row():
132
+ generate_btn = gr.Button("🚀 Generate Fast", variant="primary")
133
+ clear_btn = gr.Button("Clear")
 
 
 
 
 
 
 
134
 
135
+ with gr.Column():
136
  gallery = gr.Gallery(
137
+ label="Preview",
138
  columns=3,
139
+ height=300
 
 
140
  )
141
 
142
+ output = gr.Textbox(
143
+ label="Captions",
144
+ lines=6,
145
+ show_copy_button=True
 
 
 
 
 
 
 
 
 
 
146
  )
147
 
148
+ # Events
149
+ file_input.change(preview_images_fast, file_input, gallery)
150
+ generate_btn.click(process_images_fast, [file_input, max_length_slider], output)
151
+ clear_btn.click(lambda: (None, [], ""), outputs=[file_input, gallery, output])
 
 
 
 
 
 
 
152
 
 
153
  if __name__ == "__main__":
154
+ demo.launch(share=False, inbrowser=True)