Workpam commited on
Commit
0712911
·
verified ·
1 Parent(s): 096ff07

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +316 -81
app.py CHANGED
@@ -4,12 +4,23 @@ import torchvision.transforms as T
4
  from torchvision.models import detection
5
  from transformers import Blip2Processor, Blip2ForConditionalGeneration
6
  import gradio as gr
 
 
 
 
7
 
8
- # Device setup
 
 
 
 
9
  device = "cuda" if torch.cuda.is_available() else "cpu"
 
 
 
 
10
 
11
- # 1) Object detector (Faster R-CNN pretrained on COCO)
12
- detector = detection.fasterrcnn_resnet50_fpn(pretrained=True).eval().to(device)
13
  COCO_LABELS = [
14
  '__background__', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat',
15
  'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep',
@@ -21,93 +32,317 @@ COCO_LABELS = [
21
  'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'
22
  ]
23
 
24
- # 2) Caption model (BLIP-2 FLAN-T5 XL)
25
- processor = Blip2Processor.from_pretrained("Salesforce/blip2-flan-t5-xl", torch_dtype=torch.float16)
26
- model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-flan-t5-xl", torch_dtype=torch.float16).to(device)
27
-
28
- # Preprocessing transform
29
- transform = T.Compose([
30
- T.Resize((384, 384)),
31
- T.ToTensor(),
32
- ])
33
-
34
- # Core generation function
35
- def generate_descriptive_text(img_path, max_new_tokens=32):
36
- # Load and preprocess image
37
- img = Image.open(img_path).convert("RGB")
38
- tensor_img = transform(img).unsqueeze(0).to(device)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
 
40
- # Detect objects with high confidence
41
- with torch.no_grad():
42
- preds = detector(tensor_img)[0]
43
- labels = [
44
- COCO_LABELS[label_idx]
45
- for label_idx, score in zip(preds["labels"].cpu().tolist(), preds["scores"].cpu().tolist())
46
- if score > 0.6
47
- ]
48
- labels = sorted(set(labels))
49
-
50
- # Build prompt grounding
51
- prefix = "Objects: " + (", ".join(labels) if labels else "none detected") + "."
52
- instruction = "Describe this image factually and include key details like colors, materials, function or context."
53
-
54
- # Tokenize and generate
55
- inputs = processor(images=img, text=prefix + " " + instruction, return_tensors="pt").to(device)
56
- outputs = model.generate(
57
- **inputs,
58
- max_new_tokens=max_new_tokens,
59
- num_beams=3,
60
- no_repeat_ngram_size=2,
61
- early_stopping=True
62
- )
63
- return processor.decode(outputs[0], skip_special_tokens=True).strip()
64
 
65
- # Wrapper for Gradio UI
66
- def generate_descriptive_texts(files, max_new_tokens):
67
- captions = []
68
- for f in files or []:
69
- cap = generate_descriptive_text(f.name, max_new_tokens=int(max_new_tokens))
70
- captions.append(cap.capitalize())
71
- return "\n\n".join(captions)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
72
 
73
- # --- Gradio Interface ---
74
- with gr.Blocks(title="Descriptive Text Generator") as demo:
75
- gr.Markdown("## Descriptive Text Generator\nGrounded, factually accurate, detailed descriptions.")
 
 
 
 
 
 
 
 
 
 
 
 
 
76
 
77
- file_input = gr.File(
78
- label="Upload Images", file_count="multiple", file_types=["image"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
79
  )
80
- gallery = gr.Gallery(label="Image Preview", columns=4)
81
- slider = gr.Slider(
82
- minimum=16, maximum=64, step=8, value=32, label="Max New Tokens"
83
- )
84
- run_btn = gr.Button("Generate")
85
- clear_btn = gr.Button("Clear All")
86
- output = gr.Textbox(
87
- label="Generated Descriptions",
88
- lines=6,
89
- show_copy_button=True
90
- )
91
-
92
- # Preview uploaded images
93
  file_input.change(
94
- fn=lambda files: [Image.open(f.name).convert("RGB") for f in files or []],
95
  inputs=file_input,
96
- outputs=gallery
 
97
  )
98
-
99
- # Generate descriptions
100
- run_btn.click(
101
- fn=generate_descriptive_texts,
102
- inputs=[file_input, slider],
103
- outputs=output,
104
- queue=True
105
  )
106
-
107
- # Clear all inputs and outputs
108
  clear_btn.click(
109
- fn=lambda: ([], [], 32, ""),
110
- outputs=[file_input, gallery, slider, output]
111
  )
112
 
113
- demo.launch()
 
 
 
 
 
 
 
 
 
 
4
  from torchvision.models import detection
5
  from transformers import Blip2Processor, Blip2ForConditionalGeneration
6
  import gradio as gr
7
+ import logging
8
+ from typing import List, Optional
9
+ import gc
10
+ from functools import lru_cache
11
 
12
+ # Setup logging
13
+ logging.basicConfig(level=logging.INFO)
14
+ logger = logging.getLogger(__name__)
15
+
16
+ # Device setup with memory optimization
17
  device = "cuda" if torch.cuda.is_available() else "cpu"
18
+ if device == "cuda":
19
+ torch.cuda.empty_cache()
20
+ # Enable memory efficient attention if available
21
+ torch.backends.cuda.enable_math_sdp(True)
22
 
23
+ # COCO class labels
 
24
  COCO_LABELS = [
25
  '__background__', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat',
26
  'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep',
 
32
  'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'
33
  ]
34
 
35
+ class OptimizedImageCaptioner:
36
+ def __init__(self):
37
+ self.device = device
38
+ self.detector = None
39
+ self.processor = None
40
+ self.model = None
41
+ self._load_models()
42
+
43
+ def _load_models(self):
44
+ """Load models with error handling and optimization"""
45
+ try:
46
+ # Load object detector with optimizations
47
+ logger.info("Loading object detection model...")
48
+ self.detector = detection.fasterrcnn_resnet50_fpn(weights='DEFAULT')
49
+ self.detector.eval()
50
+ self.detector.to(self.device)
51
+
52
+ # Optimize detector for inference
53
+ if hasattr(torch.jit, 'optimize_for_inference'):
54
+ self.detector = torch.jit.optimize_for_inference(torch.jit.script(self.detector))
55
+
56
+ # Load BLIP-2 with memory optimizations
57
+ logger.info("Loading BLIP-2 model...")
58
+ model_name = "Salesforce/blip2-flan-t5-xl"
59
+
60
+ # Use appropriate dtype based on device
61
+ dtype = torch.float16 if self.device == "cuda" else torch.float32
62
+
63
+ self.processor = Blip2Processor.from_pretrained(
64
+ model_name,
65
+ torch_dtype=dtype,
66
+ low_cpu_mem_usage=True
67
+ )
68
+
69
+ self.model = Blip2ForConditionalGeneration.from_pretrained(
70
+ model_name,
71
+ torch_dtype=dtype,
72
+ low_cpu_mem_usage=True,
73
+ device_map="auto" if self.device == "cuda" else None
74
+ )
75
+
76
+ if self.device != "cuda":
77
+ self.model.to(self.device)
78
+
79
+ self.model.eval()
80
+
81
+ # Enable gradient checkpointing for memory efficiency
82
+ if hasattr(self.model, 'gradient_checkpointing_enable'):
83
+ self.model.gradient_checkpointing_enable()
84
+
85
+ logger.info("Models loaded successfully!")
86
+
87
+ except Exception as e:
88
+ logger.error(f"Error loading models: {e}")
89
+ raise
90
+
91
+ @lru_cache(maxsize=32)
92
+ def _get_transform(self, size: int = 384):
93
+ """Cached transform to avoid recreation"""
94
+ return T.Compose([
95
+ T.Resize((size, size), interpolation=T.InterpolationMode.BILINEAR),
96
+ T.ToTensor(),
97
+ T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
98
+ ])
99
+
100
+ def detect_objects(self, image_tensor: torch.Tensor, confidence_threshold: float = 0.7) -> List[str]:
101
+ """Optimized object detection with better filtering"""
102
+ try:
103
+ with torch.no_grad():
104
+ predictions = self.detector(image_tensor)[0]
105
+
106
+ # Filter by confidence and remove duplicates
107
+ detected_objects = []
108
+ seen_labels = set()
109
+
110
+ for label_idx, score in zip(predictions["labels"].cpu().tolist(),
111
+ predictions["scores"].cpu().tolist()):
112
+ if score > confidence_threshold and label_idx < len(COCO_LABELS):
113
+ label = COCO_LABELS[label_idx]
114
+ if label != '__background__' and label not in seen_labels:
115
+ detected_objects.append((label, score))
116
+ seen_labels.add(label)
117
+
118
+ # Sort by confidence and return top objects
119
+ detected_objects.sort(key=lambda x: x[1], reverse=True)
120
+ return [obj[0] for obj in detected_objects[:10]] # Limit to top 10
121
+
122
+ except Exception as e:
123
+ logger.warning(f"Object detection failed: {e}")
124
+ return []
125
+
126
+ def generate_caption(self, image: Image.Image, detected_objects: List[str],
127
+ max_new_tokens: int = 32) -> str:
128
+ """Generate caption with improved prompting"""
129
+ try:
130
+ # Create context-aware prompt
131
+ if detected_objects:
132
+ objects_str = ", ".join(detected_objects[:5]) # Limit for token efficiency
133
+ prompt = f"Objects detected: {objects_str}. Describe this image with specific details about colors, materials, actions, and setting:"
134
+ else:
135
+ prompt = "Describe this image with specific details about what you see, including colors, materials, actions, and setting:"
136
+
137
+ # Process inputs
138
+ inputs = self.processor(
139
+ images=image,
140
+ text=prompt,
141
+ return_tensors="pt",
142
+ padding=True,
143
+ truncation=True,
144
+ max_length=512
145
+ ).to(self.device)
146
+
147
+ # Generate with optimized parameters
148
+ with torch.no_grad():
149
+ outputs = self.model.generate(
150
+ **inputs,
151
+ max_new_tokens=max_new_tokens,
152
+ num_beams=3,
153
+ do_sample=False, # Deterministic for consistency
154
+ no_repeat_ngram_size=3,
155
+ early_stopping=True,
156
+ pad_token_id=self.processor.tokenizer.pad_token_id,
157
+ eos_token_id=self.processor.tokenizer.eos_token_id,
158
+ length_penalty=1.0,
159
+ repetition_penalty=1.1
160
+ )
161
+
162
+ # Decode and clean up
163
+ caption = self.processor.decode(outputs[0], skip_special_tokens=True)
164
+
165
+ # Remove the prompt from the output if it's included
166
+ if prompt in caption:
167
+ caption = caption.replace(prompt, "").strip()
168
+
169
+ return caption.strip() if caption.strip() else "Unable to generate description."
170
+
171
+ except Exception as e:
172
+ logger.error(f"Caption generation failed: {e}")
173
+ return f"Error generating caption: {str(e)}"
174
+
175
+ def process_image(self, image_path: str, max_new_tokens: int = 32) -> str:
176
+ """Main processing pipeline with error handling"""
177
+ try:
178
+ # Load and validate image
179
+ image = Image.open(image_path).convert("RGB")
180
+
181
+ # Check image size and resize if too large
182
+ max_size = 1024
183
+ if max(image.size) > max_size:
184
+ image.thumbnail((max_size, max_size), Image.Resampling.LANCZOS)
185
+
186
+ # Prepare image for object detection
187
+ transform = self._get_transform()
188
+ image_tensor = transform(image).unsqueeze(0).to(self.device)
189
+
190
+ # Detect objects
191
+ detected_objects = self.detect_objects(image_tensor)
192
+
193
+ # Generate caption
194
+ caption = self.generate_caption(image, detected_objects, max_new_tokens)
195
+
196
+ # Clean up GPU memory
197
+ if self.device == "cuda":
198
+ del image_tensor
199
+ torch.cuda.empty_cache()
200
+
201
+ return caption.capitalize() if caption else "No description available."
202
+
203
+ except Exception as e:
204
+ logger.error(f"Error processing image {image_path}: {e}")
205
+ return f"Error processing image: {str(e)}"
206
 
207
+ # Global captioner instance
208
+ captioner = OptimizedImageCaptioner()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
209
 
210
+ def process_multiple_images(files: Optional[List], max_new_tokens: int) -> str:
211
+ """Process multiple images with progress tracking"""
212
+ if not files:
213
+ return "No images uploaded."
214
+
215
+ results = []
216
+ total_files = len(files)
217
+
218
+ for i, file in enumerate(files):
219
+ try:
220
+ logger.info(f"Processing image {i+1}/{total_files}: {file.name}")
221
+ caption = captioner.process_image(file.name, max_new_tokens=int(max_new_tokens))
222
+ results.append(f"Image {i+1}: {caption}")
223
+
224
+ # Memory cleanup between images
225
+ if captioner.device == "cuda" and i % 3 == 0: # Every 3 images
226
+ torch.cuda.empty_cache()
227
+ gc.collect()
228
+
229
+ except Exception as e:
230
+ logger.error(f"Failed to process {file.name}: {e}")
231
+ results.append(f"Image {i+1}: Error - {str(e)}")
232
+
233
+ return "\n\n".join(results)
234
 
235
+ def preview_images(files):
236
+ """Preview uploaded images with error handling"""
237
+ if not files:
238
+ return []
239
+
240
+ previews = []
241
+ for file in files[:20]: # Limit previews to prevent UI overload
242
+ try:
243
+ img = Image.open(file.name).convert("RGB")
244
+ # Thumbnail for preview
245
+ img.thumbnail((300, 300), Image.Resampling.LANCZOS)
246
+ previews.append(img)
247
+ except Exception as e:
248
+ logger.warning(f"Failed to preview {file.name}: {e}")
249
+
250
+ return previews
251
 
252
+ # Gradio Interface with improved design
253
+ with gr.Blocks(
254
+ title="Advanced Image Description Generator",
255
+ theme=gr.themes.Soft(),
256
+ css="""
257
+ .gradio-container {
258
+ max-width: 1200px !important;
259
+ }
260
+ """
261
+ ) as demo:
262
+
263
+ gr.Markdown("""
264
+ # 🖼️ Advanced Image Description Generator
265
+
266
+ Generate detailed, accurate descriptions of your images using state-of-the-art AI models.
267
+ - **Object Detection**: Identifies objects in images for better context
268
+ - **Advanced Captioning**: Uses BLIP-2 for detailed, natural descriptions
269
+ - **Batch Processing**: Handle multiple images at once
270
+ """)
271
+
272
+ with gr.Row():
273
+ with gr.Column(scale=1):
274
+ file_input = gr.File(
275
+ label="📁 Upload Images",
276
+ file_count="multiple",
277
+ file_types=["image"],
278
+ height=200
279
+ )
280
+
281
+ with gr.Row():
282
+ max_tokens_slider = gr.Slider(
283
+ minimum=16,
284
+ maximum=100,
285
+ step=8,
286
+ value=48,
287
+ label="🎯 Description Length (tokens)",
288
+ info="Higher = more detailed descriptions"
289
+ )
290
+
291
+ with gr.Row():
292
+ generate_btn = gr.Button(
293
+ "✨ Generate Descriptions",
294
+ variant="primary",
295
+ size="lg"
296
+ )
297
+ clear_btn = gr.Button(
298
+ "🗑️ Clear All",
299
+ variant="secondary"
300
+ )
301
+
302
+ with gr.Column(scale=2):
303
+ gallery = gr.Gallery(
304
+ label="🖼️ Image Preview",
305
+ columns=3,
306
+ rows=2,
307
+ height=400,
308
+ object_fit="contain"
309
+ )
310
+
311
+ output_textbox = gr.Textbox(
312
+ label="📝 Generated Descriptions",
313
+ lines=8,
314
+ max_lines=20,
315
+ show_copy_button=True,
316
+ placeholder="Descriptions will appear here after processing..."
317
  )
318
+
319
+ # Event handlers
 
 
 
 
 
 
 
 
 
 
 
320
  file_input.change(
321
+ fn=preview_images,
322
  inputs=file_input,
323
+ outputs=gallery,
324
+ show_progress=True
325
  )
326
+
327
+ generate_btn.click(
328
+ fn=process_multiple_images,
329
+ inputs=[file_input, max_tokens_slider],
330
+ outputs=output_textbox,
331
+ show_progress=True
 
332
  )
333
+
 
334
  clear_btn.click(
335
+ fn=lambda: (None, [], 48, ""),
336
+ outputs=[file_input, gallery, max_tokens_slider, output_textbox]
337
  )
338
 
339
+ # Launch configuration
340
+ if __name__ == "__main__":
341
+ demo.launch(
342
+ server_name="0.0.0.0",
343
+ server_port=7860,
344
+ share=False,
345
+ debug=False,
346
+ show_error=True,
347
+ inbrowser=True
348
+ )