Workpam commited on
Commit
aa8c291
·
verified ·
1 Parent(s): f1a734d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +129 -84
app.py CHANGED
@@ -385,46 +385,76 @@ def enhanced_zero_shot_classification(image: Image.Image):
385
 
386
  return results
387
 
388
- # === Enhanced BLIP captioning ===
389
  def get_enhanced_caption(image: Image.Image):
390
- """Generate multiple captions with different prompts and combine insights."""
391
- prompts = [
392
- None, # Unconditional
393
- "Describe this product in detail including its brand, type, colors, and any visible text or logos.",
394
- "What are the key visual features and characteristics of this item?",
395
- "Provide a comprehensive description of this object including materials, colors, and design elements."
396
  ]
397
 
398
  captions = []
399
- for prompt in prompts:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
400
  try:
401
- if prompt is None:
402
- inputs = blip_processor(image, return_tensors="pt").to(device)
403
- else:
404
- inputs = blip_processor(image, text=prompt, return_tensors="pt").to(device)
405
-
406
  with torch.inference_mode():
407
  out = blip_model.generate(
408
  **inputs,
409
- max_new_tokens=80,
410
  num_beams=5,
411
  do_sample=True,
412
  temperature=0.7,
 
413
  early_stopping=True,
414
- no_repeat_ngram_size=3
 
415
  )
416
 
417
  caption = blip_processor.decode(out[0], skip_special_tokens=True).strip()
418
- if prompt and prompt.lower() in caption.lower():
419
- continue # Skip if caption just repeats the prompt
420
- captions.append(caption)
421
 
 
 
 
 
 
 
 
422
  except Exception as e:
423
- print(f"Caption generation error: {e}")
424
  continue
425
 
426
  # Return the longest, most detailed caption
427
- return max(captions, key=len) if captions else "Unable to generate caption"
 
 
 
 
 
 
428
 
429
  # === Comprehensive analysis pipeline ===
430
  def comprehensive_image_analysis(image: Image.Image):
@@ -456,73 +486,89 @@ def comprehensive_image_analysis(image: Image.Image):
456
 
457
  return analysis
458
 
459
- # === Alt-text generation ===
460
- def build_comprehensive_alt_text(analysis):
461
- """Build detailed alt-text from comprehensive analysis."""
462
- parts = []
463
 
464
- # Main caption
465
- if analysis.get('caption'):
466
- parts.append(f"Description: {analysis['caption']}")
467
 
468
- # Product type (from classification)
469
- products = analysis.get('classification', {}).get('products', [])
470
- if products and products[0][1] > 0.3: # Confidence threshold
471
- parts.append(f"Product type: {products[0][0]}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
472
 
473
- # Brand (from classification and VQA)
 
 
 
 
 
 
 
 
 
 
474
  brands = analysis.get('classification', {}).get('brands', [])
475
  brand_vqa = analysis.get('vqa', {}).get('brand is visible in this image', '')
476
 
477
- brand_text = ""
478
- if brands and brands[0][1] > 0.25 and brands[0][0] not in ['unknown', 'generic', 'no brand visible']:
479
- brand_text = brands[0][0]
480
  elif brand_vqa and len(brand_vqa) > 2 and brand_vqa.lower() not in ['no', 'none', 'unknown']:
481
- brand_text = brand_vqa
482
-
483
- if brand_text:
484
- parts.append(f"Brand: {brand_text}")
485
-
486
- # Characters/People
487
- characters = analysis.get('classification', {}).get('characters', [])
488
- if characters and characters[0][1] > 0.4:
489
- parts.append(f"Features: {characters[0][0]}")
490
-
491
- # Colors
492
- colors = analysis.get('colors', [])
493
- if colors:
494
- color_text = ", ".join(colors[:3]) # Top 3 colors
495
- parts.append(f"Dominant colors: {color_text}")
496
 
497
- # OCR text
498
- ocr_text = analysis.get('ocr_text', '')
499
- if ocr_text:
500
- parts.append(f"Visible text: \"{ocr_text}\"")
501
 
502
- # Key details
503
- details = analysis.get('classification', {}).get('details', [])
504
- high_conf_details = [detail for detail, conf in details if conf > 0.4]
505
- if high_conf_details:
506
- parts.append(f"Visual elements: {', '.join(high_conf_details[:3])}")
 
507
 
508
- # VQA insights
509
  vqa = analysis.get('vqa', {})
510
  material = vqa.get('material does this product appear to be made of', '')
511
- if material and len(material) > 2:
512
- parts.append(f"Material: {material}")
513
-
514
- background = vqa.get('background of this image', '')
515
- if background and len(background) > 5 and background.lower() not in ['white', 'plain', 'simple']:
516
- parts.append(f"Background: {background}")
 
 
 
 
 
517
 
518
- # Combine all parts
519
- alt_text = "; ".join(parts)
 
 
520
 
521
  # Ensure proper ending
522
- if not alt_text.endswith('.'):
523
- alt_text += '.'
524
 
525
- return alt_text
526
 
527
  # === Initialize embeddings ===
528
  print("Precomputing embeddings...")
@@ -539,7 +585,7 @@ def handle_single_image(file):
539
 
540
  try:
541
  analysis = comprehensive_image_analysis(image)
542
- alt_text = build_comprehensive_alt_text(analysis)
543
  return alt_text
544
  except Exception as e:
545
  return f"Analysis failed: {e}"
@@ -596,18 +642,17 @@ with gr.Blocks(
596
  ) as demo:
597
 
598
  gr.Markdown("""
599
- # 🔍 Advanced Product Alt-Text Generator
600
-
601
- **Enhanced with comprehensive analysis capabilities:**
602
- - ✅ **Multi-model approach**: BLIP captioning + VQA + CLIP classification
603
- - ✅ **Extensive object recognition**: 100+ product types, 50+ brands
604
- - ✅ **Character & person detection**: Identifies people, mascots, characters
605
- - ✅ **Advanced OCR**: Multi-language text extraction with preprocessing
606
- - ✅ **Color analysis**: Dominant color extraction using K-means clustering
607
- - ✅ **Visual question answering**: Asks specific questions about image content
608
- - ✅ **Material & texture recognition**: Identifies materials and surface properties
609
-
610
- Upload product images to generate detailed, accurate alt-text descriptions.
611
  """)
612
 
613
  with gr.Row():
 
385
 
386
  return results
387
 
388
+ # === Enhanced BLIP captioning with detailed prompts ===
389
  def get_enhanced_caption(image: Image.Image):
390
+ """Generate detailed narrative captions with specific prompts."""
391
+ detailed_prompts = [
392
+ "Provide a detailed, comprehensive description of this image including all visual elements, characters, objects, text, colors, setting, and artistic style.",
393
+ "Describe this image in great detail, mentioning the title, characters' appearance, clothing, expressions, objects they're holding, background elements, and any text visible.",
394
+ "Write a thorough description of this cover art or product image, including character details, clothing, facial expressions, objects, background setting, colors, and any visible text or branding.",
395
+ "Analyze this image completely: describe the main subject, their appearance and clothing, any objects or accessories, the background setting, color scheme, artistic style, and all visible text.",
396
  ]
397
 
398
  captions = []
399
+
400
+ # Try unconditional first for baseline
401
+ try:
402
+ inputs = blip_processor(image, return_tensors="pt").to(device)
403
+ with torch.inference_mode():
404
+ out = blip_model.generate(
405
+ **inputs,
406
+ max_new_tokens=150, # Increased for more detail
407
+ num_beams=6,
408
+ do_sample=True,
409
+ temperature=0.8,
410
+ top_p=0.9,
411
+ early_stopping=True,
412
+ no_repeat_ngram_size=2,
413
+ repetition_penalty=1.1
414
+ )
415
+ caption = blip_processor.decode(out[0], skip_special_tokens=True).strip()
416
+ captions.append(caption)
417
+ except Exception as e:
418
+ print(f"Unconditional caption error: {e}")
419
+
420
+ # Try detailed prompts
421
+ for prompt in detailed_prompts[:2]: # Use first 2 prompts to avoid overload
422
  try:
423
+ inputs = blip_processor(image, text=prompt, return_tensors="pt").to(device)
 
 
 
 
424
  with torch.inference_mode():
425
  out = blip_model.generate(
426
  **inputs,
427
+ max_new_tokens=120,
428
  num_beams=5,
429
  do_sample=True,
430
  temperature=0.7,
431
+ top_p=0.85,
432
  early_stopping=True,
433
+ no_repeat_ngram_size=3,
434
+ repetition_penalty=1.15
435
  )
436
 
437
  caption = blip_processor.decode(out[0], skip_special_tokens=True).strip()
 
 
 
438
 
439
+ # Filter out prompts that got repeated in output
440
+ if not any(phrase in caption.lower() for phrase in [
441
+ "provide a detailed", "describe this image", "write a thorough",
442
+ "analyze this image", "comprehensive description"
443
+ ]):
444
+ captions.append(caption)
445
+
446
  except Exception as e:
447
+ print(f"Detailed caption error: {e}")
448
  continue
449
 
450
  # Return the longest, most detailed caption
451
+ if captions:
452
+ best_caption = max(captions, key=len)
453
+ # Clean up the caption
454
+ best_caption = re.sub(r'^(the|a|an)\s+', '', best_caption, flags=re.IGNORECASE).strip()
455
+ return best_caption.capitalize()
456
+
457
+ return "Unable to generate detailed caption"
458
 
459
  # === Comprehensive analysis pipeline ===
460
  def comprehensive_image_analysis(image: Image.Image):
 
486
 
487
  return analysis
488
 
489
+ # === Narrative alt-text generation ===
490
+ def build_narrative_alt_text(analysis):
491
+ """Build detailed narrative alt-text from comprehensive analysis."""
 
492
 
493
+ # Start with the main detailed caption
494
+ main_description = analysis.get('caption', '')
 
495
 
496
+ # Enhance with additional details from other analyses
497
+ enhancement_parts = []
498
+
499
+ # Add OCR text information naturally
500
+ ocr_text = analysis.get('ocr_text', '')
501
+ if ocr_text and len(ocr_text.strip()) > 2:
502
+ # Try to integrate OCR text naturally into description
503
+ if 'vol' in ocr_text.lower() or 'volume' in ocr_text.lower():
504
+ enhancement_parts.append(f"Text visible includes: \"{ocr_text}\"")
505
+ elif any(brand in ocr_text.lower() for brand in ['manga', 'comic', 'book']):
506
+ enhancement_parts.append(f"The title text reads: \"{ocr_text}\"")
507
+ else:
508
+ enhancement_parts.append(f"Visible text: \"{ocr_text}\"")
509
+
510
+ # Add character/person details if detected with high confidence
511
+ characters = analysis.get('classification', {}).get('characters', [])
512
+ if characters and characters[0][1] > 0.5: # High confidence threshold
513
+ char_type = characters[0][0]
514
+ if char_type not in ['person'] and 'character' in char_type:
515
+ enhancement_parts.append(f"The image features {char_type} art style")
516
 
517
+ # Add dominant colors in a natural way
518
+ colors = analysis.get('colors', [])
519
+ if colors and len(colors) >= 2:
520
+ color_desc = f"The color palette prominently features {colors[0]}"
521
+ if len(colors) > 1:
522
+ color_desc += f", {colors[1]}"
523
+ if len(colors) > 2:
524
+ color_desc += f", and {colors[2]}"
525
+ enhancement_parts.append(color_desc)
526
+
527
+ # Add brand information if detected and relevant
528
  brands = analysis.get('classification', {}).get('brands', [])
529
  brand_vqa = analysis.get('vqa', {}).get('brand is visible in this image', '')
530
 
531
+ brand_info = ""
532
+ if brands and brands[0][1] > 0.3 and brands[0][0] not in ['unknown', 'generic', 'no brand visible']:
533
+ brand_info = brands[0][0]
534
  elif brand_vqa and len(brand_vqa) > 2 and brand_vqa.lower() not in ['no', 'none', 'unknown']:
535
+ brand_info = brand_vqa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
536
 
537
+ if brand_info and brand_info.lower() not in main_description.lower():
538
+ enhancement_parts.append(f"Published by {brand_info}")
 
 
539
 
540
+ # Add product type context if it adds value
541
+ products = analysis.get('classification', {}).get('products', [])
542
+ if products and products[0][1] > 0.4:
543
+ product_type = products[0][0]
544
+ if product_type in ['comic book', 'manga', 'book'] and product_type not in main_description.lower():
545
+ main_description = f"This is a {product_type} cover. " + main_description
546
 
547
+ # Add VQA insights that provide additional context
548
  vqa = analysis.get('vqa', {})
549
  material = vqa.get('material does this product appear to be made of', '')
550
+ if material and len(material) > 3 and material.lower() not in ['paper', 'unknown']:
551
+ enhancement_parts.append(f"The material appears to be {material}")
552
+
553
+ # Combine main description with enhancements
554
+ if enhancement_parts:
555
+ # Add enhancements naturally to the main description
556
+ enhanced_description = main_description
557
+ for enhancement in enhancement_parts:
558
+ enhanced_description += f" {enhancement}."
559
+ else:
560
+ enhanced_description = main_description
561
 
562
+ # Clean up the final description
563
+ enhanced_description = re.sub(r'\s+', ' ', enhanced_description) # Remove extra spaces
564
+ enhanced_description = re.sub(r'\.+', '.', enhanced_description) # Remove multiple periods
565
+ enhanced_description = enhanced_description.strip()
566
 
567
  # Ensure proper ending
568
+ if not enhanced_description.endswith('.'):
569
+ enhanced_description += '.'
570
 
571
+ return enhanced_description
572
 
573
  # === Initialize embeddings ===
574
  print("Precomputing embeddings...")
 
585
 
586
  try:
587
  analysis = comprehensive_image_analysis(image)
588
+ alt_text = build_narrative_alt_text(analysis)
589
  return alt_text
590
  except Exception as e:
591
  return f"Analysis failed: {e}"
 
642
  ) as demo:
643
 
644
  gr.Markdown("""
645
+ # 🖼️ Narrative Alt-Text Generator
646
+
647
+ **Generates detailed, flowing descriptions like human-written alt-text:**
648
+ - ✅ **Detailed narrative style**: Natural, paragraph-form descriptions
649
+ - ✅ **Character analysis**: Detailed appearance, clothing, expressions
650
+ - ✅ **Scene description**: Background, setting, artistic elements
651
+ - ✅ **Text integration**: Naturally incorporates visible text and titles
652
+ - ✅ **Artistic context**: Identifies manga/comic/book covers with style notes
653
+ - ✅ **Enhanced BLIP**: Uses advanced prompting for richer descriptions
654
+
655
+ Upload images to generate detailed, narrative alt-text descriptions similar to professional accessibility descriptions.
 
656
  """)
657
 
658
  with gr.Row():