Update app.py
Browse files
app.py
CHANGED
|
@@ -24,14 +24,20 @@ blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning
|
|
| 24 |
blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
|
| 25 |
blip_model.to(device).eval()
|
| 26 |
|
| 27 |
-
# BLIP for VQA (Visual Question Answering)
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
blip_vqa_model.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
if device.type == "cuda":
|
| 33 |
blip_model = blip_model.half()
|
| 34 |
-
|
|
|
|
| 35 |
|
| 36 |
# CLIP for zero-shot classification
|
| 37 |
clip_model, _, clip_preprocess = open_clip.create_model_and_transforms("ViT-L-14", pretrained="openai")
|
|
@@ -226,6 +232,9 @@ def extract_dominant_colors(image: Image.Image, n_colors=3):
|
|
| 226 |
# === Enhanced VQA for detailed analysis ===
|
| 227 |
def ask_visual_question(image: Image.Image, question: str):
|
| 228 |
"""Use BLIP VQA to ask specific questions about the image."""
|
|
|
|
|
|
|
|
|
|
| 229 |
try:
|
| 230 |
inputs = blip_vqa_processor(image, question, return_tensors="pt").to(device)
|
| 231 |
with torch.inference_mode():
|
|
@@ -238,6 +247,9 @@ def ask_visual_question(image: Image.Image, question: str):
|
|
| 238 |
|
| 239 |
def analyze_with_vqa(image: Image.Image):
|
| 240 |
"""Analyze image using multiple VQA questions."""
|
|
|
|
|
|
|
|
|
|
| 241 |
questions = [
|
| 242 |
"What type of product is this?",
|
| 243 |
"What brand is visible in this image?",
|
|
@@ -427,8 +439,11 @@ def comprehensive_image_analysis(image: Image.Image):
|
|
| 427 |
# Enhanced captioning
|
| 428 |
analysis['caption'] = get_enhanced_caption(image)
|
| 429 |
|
| 430 |
-
# VQA analysis
|
| 431 |
-
|
|
|
|
|
|
|
|
|
|
| 432 |
|
| 433 |
# Zero-shot classification
|
| 434 |
analysis['classification'] = enhanced_zero_shot_classification(image)
|
|
|
|
| 24 |
blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
|
| 25 |
blip_model.to(device).eval()
|
| 26 |
|
| 27 |
+
# BLIP for VQA (Visual Question Answering) - using base model for stability
|
| 28 |
+
try:
|
| 29 |
+
blip_vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
|
| 30 |
+
blip_vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
|
| 31 |
+
blip_vqa_model.to(device).eval()
|
| 32 |
+
VQA_AVAILABLE = True
|
| 33 |
+
except Exception as e:
|
| 34 |
+
print(f"VQA model not available: {e}")
|
| 35 |
+
VQA_AVAILABLE = False
|
| 36 |
|
| 37 |
if device.type == "cuda":
|
| 38 |
blip_model = blip_model.half()
|
| 39 |
+
if VQA_AVAILABLE:
|
| 40 |
+
blip_vqa_model = blip_vqa_model.half()
|
| 41 |
|
| 42 |
# CLIP for zero-shot classification
|
| 43 |
clip_model, _, clip_preprocess = open_clip.create_model_and_transforms("ViT-L-14", pretrained="openai")
|
|
|
|
| 232 |
# === Enhanced VQA for detailed analysis ===
|
| 233 |
def ask_visual_question(image: Image.Image, question: str):
|
| 234 |
"""Use BLIP VQA to ask specific questions about the image."""
|
| 235 |
+
if not VQA_AVAILABLE:
|
| 236 |
+
return ""
|
| 237 |
+
|
| 238 |
try:
|
| 239 |
inputs = blip_vqa_processor(image, question, return_tensors="pt").to(device)
|
| 240 |
with torch.inference_mode():
|
|
|
|
| 247 |
|
| 248 |
def analyze_with_vqa(image: Image.Image):
|
| 249 |
"""Analyze image using multiple VQA questions."""
|
| 250 |
+
if not VQA_AVAILABLE:
|
| 251 |
+
return {}
|
| 252 |
+
|
| 253 |
questions = [
|
| 254 |
"What type of product is this?",
|
| 255 |
"What brand is visible in this image?",
|
|
|
|
| 439 |
# Enhanced captioning
|
| 440 |
analysis['caption'] = get_enhanced_caption(image)
|
| 441 |
|
| 442 |
+
# VQA analysis (if available)
|
| 443 |
+
if VQA_AVAILABLE:
|
| 444 |
+
analysis['vqa'] = analyze_with_vqa(image)
|
| 445 |
+
else:
|
| 446 |
+
analysis['vqa'] = {}
|
| 447 |
|
| 448 |
# Zero-shot classification
|
| 449 |
analysis['classification'] = enhanced_zero_shot_classification(image)
|