Workpam commited on
Commit
f1a734d
·
verified ·
1 Parent(s): 6ab0927

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +22 -7
app.py CHANGED
@@ -24,14 +24,20 @@ blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning
24
  blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
25
  blip_model.to(device).eval()
26
 
27
- # BLIP for VQA (Visual Question Answering)
28
- blip_vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
29
- blip_vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
30
- blip_vqa_model.to(device).eval()
 
 
 
 
 
31
 
32
  if device.type == "cuda":
33
  blip_model = blip_model.half()
34
- blip_vqa_model = blip_vqa_model.half()
 
35
 
36
  # CLIP for zero-shot classification
37
  clip_model, _, clip_preprocess = open_clip.create_model_and_transforms("ViT-L-14", pretrained="openai")
@@ -226,6 +232,9 @@ def extract_dominant_colors(image: Image.Image, n_colors=3):
226
  # === Enhanced VQA for detailed analysis ===
227
  def ask_visual_question(image: Image.Image, question: str):
228
  """Use BLIP VQA to ask specific questions about the image."""
 
 
 
229
  try:
230
  inputs = blip_vqa_processor(image, question, return_tensors="pt").to(device)
231
  with torch.inference_mode():
@@ -238,6 +247,9 @@ def ask_visual_question(image: Image.Image, question: str):
238
 
239
  def analyze_with_vqa(image: Image.Image):
240
  """Analyze image using multiple VQA questions."""
 
 
 
241
  questions = [
242
  "What type of product is this?",
243
  "What brand is visible in this image?",
@@ -427,8 +439,11 @@ def comprehensive_image_analysis(image: Image.Image):
427
  # Enhanced captioning
428
  analysis['caption'] = get_enhanced_caption(image)
429
 
430
- # VQA analysis
431
- analysis['vqa'] = analyze_with_vqa(image)
 
 
 
432
 
433
  # Zero-shot classification
434
  analysis['classification'] = enhanced_zero_shot_classification(image)
 
24
  blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
25
  blip_model.to(device).eval()
26
 
27
+ # BLIP for VQA (Visual Question Answering) - using base model for stability
28
+ try:
29
+ blip_vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
30
+ blip_vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
31
+ blip_vqa_model.to(device).eval()
32
+ VQA_AVAILABLE = True
33
+ except Exception as e:
34
+ print(f"VQA model not available: {e}")
35
+ VQA_AVAILABLE = False
36
 
37
  if device.type == "cuda":
38
  blip_model = blip_model.half()
39
+ if VQA_AVAILABLE:
40
+ blip_vqa_model = blip_vqa_model.half()
41
 
42
  # CLIP for zero-shot classification
43
  clip_model, _, clip_preprocess = open_clip.create_model_and_transforms("ViT-L-14", pretrained="openai")
 
232
  # === Enhanced VQA for detailed analysis ===
233
  def ask_visual_question(image: Image.Image, question: str):
234
  """Use BLIP VQA to ask specific questions about the image."""
235
+ if not VQA_AVAILABLE:
236
+ return ""
237
+
238
  try:
239
  inputs = blip_vqa_processor(image, question, return_tensors="pt").to(device)
240
  with torch.inference_mode():
 
247
 
248
  def analyze_with_vqa(image: Image.Image):
249
  """Analyze image using multiple VQA questions."""
250
+ if not VQA_AVAILABLE:
251
+ return {}
252
+
253
  questions = [
254
  "What type of product is this?",
255
  "What brand is visible in this image?",
 
439
  # Enhanced captioning
440
  analysis['caption'] = get_enhanced_caption(image)
441
 
442
+ # VQA analysis (if available)
443
+ if VQA_AVAILABLE:
444
+ analysis['vqa'] = analyze_with_vqa(image)
445
+ else:
446
+ analysis['vqa'] = {}
447
 
448
  # Zero-shot classification
449
  analysis['classification'] = enhanced_zero_shot_classification(image)