Update app.py
Browse files
app.py
CHANGED
|
@@ -6,16 +6,17 @@ import open_clip
|
|
| 6 |
import cv2
|
| 7 |
import numpy as np
|
| 8 |
import re
|
| 9 |
-
from collections import Counter
|
| 10 |
-
from transformers import BlipProcessor, BlipForConditionalGeneration, BlipForQuestionAnswering
|
| 11 |
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 12 |
from sklearn.cluster import KMeans
|
| 13 |
-
from
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
|
| 15 |
# === Device setup ===
|
| 16 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 17 |
torch.backends.cudnn.benchmark = True
|
| 18 |
-
|
| 19 |
print(f"Using device: {device}")
|
| 20 |
|
| 21 |
# === Model initialization ===
|
|
@@ -24,7 +25,7 @@ blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning
|
|
| 24 |
blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
|
| 25 |
blip_model.to(device).eval()
|
| 26 |
|
| 27 |
-
# BLIP for VQA (
|
| 28 |
try:
|
| 29 |
blip_vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
|
| 30 |
blip_vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
|
|
@@ -45,307 +46,216 @@ clip_model.to(device).eval()
|
|
| 45 |
if device.type == "cuda":
|
| 46 |
clip_model = clip_model.half()
|
| 47 |
|
| 48 |
-
# OCR reader
|
| 49 |
ocr_reader = easyocr.Reader(["en", "es", "fr", "de", "it"], gpu=(device.type == "cuda"))
|
| 50 |
|
| 51 |
-
# ===
|
| 52 |
-
COMPREHENSIVE_PRODUCT_LABELS = [
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
"water bottle", "soda can", "beer bottle", "wine bottle", "coffee mug", "tea cup",
|
| 71 |
-
"food package", "cereal box", "snack bag", "candy bar", "chocolate", "cookie package",
|
| 72 |
-
# Books & Media
|
| 73 |
-
"book", "magazine", "comic book", "DVD", "Blu-ray", "CD", "vinyl record",
|
| 74 |
-
"notebook", "journal", "planner", "calendar", "poster", "artwork", "photo frame",
|
| 75 |
-
# Sports & Fitness
|
| 76 |
-
"basketball", "football", "soccer ball", "tennis ball", "baseball", "golf ball",
|
| 77 |
-
"dumbbells", "yoga mat", "water bottle", "sports equipment", "helmet", "gloves",
|
| 78 |
-
# Home & Kitchen
|
| 79 |
-
"coffee maker", "blender", "toaster", "microwave", "pan", "pot", "plate", "bowl",
|
| 80 |
-
"cup", "glass", "utensils", "knife", "cutting board", "storage container",
|
| 81 |
-
# Art & Craft Supplies
|
| 82 |
-
"paint brush", "marker", "pen", "pencil", "crayon", "paint tube", "canvas",
|
| 83 |
-
"sketchbook", "ruler", "scissors", "glue", "tape", "stickers",
|
| 84 |
-
# Packaging & Containers
|
| 85 |
-
"box", "package", "container", "jar", "bottle", "can", "tube", "pouch", "bag"
|
| 86 |
-
]
|
| 87 |
-
|
| 88 |
-
COMPREHENSIVE_BRAND_LABELS = [
|
| 89 |
-
# Tech Brands
|
| 90 |
-
"Apple", "Samsung", "Google", "Microsoft", "Sony", "Nintendo", "PlayStation", "Xbox",
|
| 91 |
-
"HP", "Dell", "Lenovo", "ASUS", "Acer", "Canon", "Nikon", "GoPro", "DJI",
|
| 92 |
-
# Fashion & Accessories
|
| 93 |
-
"Nike", "Adidas", "Puma", "Under Armour", "Reebok", "Converse", "Vans", "New Balance",
|
| 94 |
-
"Ray-Ban", "Oakley", "Rolex", "Casio", "Fossil", "Michael Kors", "Coach", "Louis Vuitton",
|
| 95 |
-
# Beauty & Personal Care
|
| 96 |
-
"L'Oreal", "Maybelline", "MAC", "Sephora", "Clinique", "Estee Lauder", "Chanel",
|
| 97 |
-
"Dior", "YSL", "Urban Decay", "Too Faced", "NARS", "Fenty Beauty",
|
| 98 |
-
# Gaming & Toys
|
| 99 |
-
"Hasbro", "Mattel", "LEGO", "Ravensburger", "Fantasy Flight Games", "Asmodee",
|
| 100 |
-
"Days of Wonder", "CMON", "Wizards of the Coast", "Pokémon", "Yu-Gi-Oh",
|
| 101 |
-
# Food & Beverages
|
| 102 |
-
"Coca-Cola", "Pepsi", "Starbucks", "Red Bull", "Monster", "Gatorade", "Powerade",
|
| 103 |
-
"Nestle", "Hershey's", "Mars", "Ferrero", "Lindt", "Godiva",
|
| 104 |
-
# Art & Craft
|
| 105 |
-
"Crayola", "Sharpie", "Prismacolor", "Winsor & Newton", "Molotow", "Posca",
|
| 106 |
-
"Staedtler", "Faber-Castell", "Sakura", "Tombow", "Pilot", "Uni-ball",
|
| 107 |
-
# Specialty/Generic
|
| 108 |
-
"unknown", "generic", "no brand visible"
|
| 109 |
-
]
|
| 110 |
-
|
| 111 |
-
CHARACTER_LABELS = [
|
| 112 |
-
"person", "man", "woman", "child", "baby", "elderly person", "athlete", "professional",
|
| 113 |
-
"cartoon character", "anime character", "superhero", "mascot", "celebrity", "model",
|
| 114 |
-
"fantasy character", "sci-fi character", "historical figure", "fictional character"
|
| 115 |
-
]
|
| 116 |
-
|
| 117 |
-
DETAIL_LABELS = [
|
| 118 |
-
"text", "logo", "brand name", "price tag", "barcode", "QR code", "warning label",
|
| 119 |
-
"ingredients list", "nutritional information", "instructions", "serial number",
|
| 120 |
-
"model number", "size information", "color variants", "pattern", "texture",
|
| 121 |
-
"metallic finish", "matte finish", "glossy finish", "transparent", "translucent"
|
| 122 |
-
]
|
| 123 |
-
|
| 124 |
-
# === Enhanced color detection ===
|
| 125 |
-
CSS3_COLORS = {
|
| 126 |
-
# ... (same mapping as before) ...
|
| 127 |
}
|
| 128 |
|
| 129 |
-
def
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
for
|
| 133 |
-
dist = sum((a - b)**2 for a, b in zip(rgb_tuple,
|
| 134 |
-
if dist <
|
| 135 |
-
|
| 136 |
-
return
|
| 137 |
|
| 138 |
def extract_dominant_colors(image: Image.Image, n_colors=3):
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
|
|
|
|
|
|
|
|
|
| 143 |
kmeans = KMeans(n_clusters=n_colors, random_state=42, n_init=10).fit(arr)
|
| 144 |
colors = []
|
| 145 |
for center in kmeans.cluster_centers_:
|
| 146 |
rgb = tuple(map(int, center))
|
| 147 |
-
colors.append(
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
def analyze_with_vqa(image: Image.Image):
|
| 160 |
-
if not VQA_AVAILABLE: return {}
|
| 161 |
-
qs = [
|
| 162 |
-
"What type of product is this?",
|
| 163 |
-
"What brand is visible in this image?",
|
| 164 |
-
"What colors are prominent in this image?",
|
| 165 |
-
"Is there any text visible in this image?",
|
| 166 |
-
"Are there any people or characters in this image?",
|
| 167 |
-
"What is the main object in this image?",
|
| 168 |
-
"What material does this product appear to be made of?",
|
| 169 |
-
"What is the background of this image?",
|
| 170 |
-
"Are there any logos or symbols visible?",
|
| 171 |
-
"What is the shape of the main object?"
|
| 172 |
-
]
|
| 173 |
-
results = {}
|
| 174 |
-
for q in qs:
|
| 175 |
-
ans = ask_visual_question(image, q)
|
| 176 |
-
if ans and len(ans)>2:
|
| 177 |
-
key = q.lower().rstrip("?")
|
| 178 |
-
results[key] = ans
|
| 179 |
-
return results
|
| 180 |
-
|
| 181 |
-
def extract_comprehensive_ocr(image: Image.Image):
|
| 182 |
-
try:
|
| 183 |
-
arr = np.array(image)
|
| 184 |
-
gray = cv2.cvtColor(arr, cv2.COLOR_RGB2GRAY)
|
| 185 |
-
pre = []
|
| 186 |
-
pre.append(arr)
|
| 187 |
-
clahe = cv2.createCLAHE(clipLimit=2.0,tileGridSize=(8,8)).apply(gray)
|
| 188 |
-
pre.append(cv2.cvtColor(clahe,cv2.COLOR_GRAY2RGB))
|
| 189 |
-
_,th = cv2.threshold(gray,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)
|
| 190 |
-
pre.append(cv2.cvtColor(th,cv2.COLOR_GRAY2RGB))
|
| 191 |
-
texts=[]
|
| 192 |
-
for p in pre:
|
| 193 |
-
for bbox, txt, conf in ocr_reader.readtext(p,detail=1,paragraph=True):
|
| 194 |
-
if conf>0.3 and len(txt.strip())>1:
|
| 195 |
-
texts.append((txt.strip(),conf))
|
| 196 |
-
seen,set_seen=[],set()
|
| 197 |
-
for t,c in sorted(texts,key=lambda x:x[1],reverse=True):
|
| 198 |
-
cc=re.sub(r'\W+','',t.lower())
|
| 199 |
-
if cc not in set_seen and len(cc)>1:
|
| 200 |
-
seen.append(t); set_seen.add(cc)
|
| 201 |
-
return " | ".join(seen[:5])
|
| 202 |
-
except:
|
| 203 |
-
return ""
|
| 204 |
-
|
| 205 |
-
# === Zero-shot setup ===
|
| 206 |
def precompute_embeddings():
|
| 207 |
-
global
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
with torch.inference_mode():
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 217 |
|
| 218 |
def enhanced_zero_shot_classification(image: Image.Image):
|
| 219 |
with torch.inference_mode():
|
| 220 |
img_t = clip_preprocess(image).unsqueeze(0).to(device)
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
for
|
| 225 |
-
(COMPREHENSIVE_PRODUCT_LABELS,
|
| 226 |
-
(COMPREHENSIVE_BRAND_LABELS,
|
| 227 |
-
(CHARACTER_LABELS,
|
| 228 |
-
(DETAIL_LABELS,
|
| 229 |
]:
|
| 230 |
-
logits = 100.0 *
|
| 231 |
probs = torch.softmax(logits, dim=-1)
|
| 232 |
top = torch.topk(probs, topk, dim=-1)
|
| 233 |
-
|
| 234 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
|
| 236 |
-
# === Updated enhanced captioning ===
|
| 237 |
def get_enhanced_caption(image: Image.Image):
|
| 238 |
-
detailed_prompts = [
|
| 239 |
-
"Provide a detailed, comprehensive description of this image including all visual elements, characters, objects, text, colors, setting, and artistic style.",
|
| 240 |
-
"Describe this image in great detail, mentioning the title, characters' appearance, clothing, expressions, objects they're holding, background elements, and any text visible.",
|
| 241 |
-
"Write a thorough description of this cover art or product image, including character details, clothing, facial expressions, objects, background setting, colors, and any visible text or branding.",
|
| 242 |
-
"Analyze this image completely: describe the main subject, their appearance and clothing, any objects or accessories, the background setting, color scheme, artistic style, and all visible text."
|
| 243 |
-
]
|
| 244 |
-
|
| 245 |
captions = []
|
| 246 |
-
#
|
| 247 |
try:
|
| 248 |
inputs = blip_processor(image, return_tensors="pt").to(device)
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
do_sample=True,
|
| 255 |
-
temperature=0.8,
|
| 256 |
-
top_p=0.9,
|
| 257 |
-
early_stopping=True,
|
| 258 |
-
no_repeat_ngram_size=2,
|
| 259 |
-
repetition_penalty=1.1
|
| 260 |
-
)
|
| 261 |
captions.append(blip_processor.decode(out[0], skip_special_tokens=True).strip())
|
| 262 |
except Exception as e:
|
| 263 |
-
print(f"
|
| 264 |
|
| 265 |
-
#
|
| 266 |
-
for prompt in
|
| 267 |
try:
|
| 268 |
inputs = blip_processor(image, text=prompt, return_tensors="pt").to(device)
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
do_sample=True,
|
| 275 |
-
temperature=0.7,
|
| 276 |
-
top_p=0.85,
|
| 277 |
-
early_stopping=True,
|
| 278 |
-
no_repeat_ngram_size=3,
|
| 279 |
-
repetition_penalty=1.15
|
| 280 |
-
)
|
| 281 |
captions.append(blip_processor.decode(out[0], skip_special_tokens=True).strip())
|
| 282 |
except Exception as e:
|
| 283 |
-
print(f"
|
|
|
|
|
|
|
|
|
|
| 284 |
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 290 |
|
| 291 |
# === Comprehensive pipeline ===
|
| 292 |
def comprehensive_image_analysis(image: Image.Image):
|
|
|
|
| 293 |
if max(image.size) > 1024:
|
| 294 |
image = image.copy()
|
| 295 |
image.thumbnail((1024,1024), Image.Resampling.LANCZOS)
|
| 296 |
|
| 297 |
-
|
| 298 |
'caption': get_enhanced_caption(image),
|
| 299 |
-
'vqa': analyze_with_vqa(image) if VQA_AVAILABLE else {},
|
| 300 |
'classification': enhanced_zero_shot_classification(image),
|
| 301 |
'ocr_text': extract_comprehensive_ocr(image),
|
| 302 |
'colors': extract_dominant_colors(image)
|
| 303 |
}
|
| 304 |
-
return analysis
|
| 305 |
|
|
|
|
| 306 |
def build_narrative_alt_text(analysis):
|
| 307 |
-
|
| 308 |
parts = []
|
| 309 |
|
| 310 |
-
|
|
|
|
| 311 |
if ocr:
|
| 312 |
-
|
| 313 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 314 |
else:
|
| 315 |
-
|
|
|
|
| 316 |
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
|
|
|
| 320 |
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
cols_str = ", ".join(cols[:-1]) + (f", and {cols[-1]}" if len(cols)>1 else "")
|
| 324 |
-
parts.append(f'The color palette prominently features {cols_str}')
|
| 325 |
-
|
| 326 |
-
brs = analysis['classification'].get('brands',[])
|
| 327 |
-
vqa_brand = analysis['vqa'].get('what brand is visible in this image','')
|
| 328 |
-
brand = ""
|
| 329 |
-
if brs and brs[0][1]>0.3 and brs[0][0] not in ['unknown','generic','no brand visible']:
|
| 330 |
-
brand = brs[0][0]
|
| 331 |
-
elif vqa_brand and vqa_brand.lower() not in ['no','none','unknown']:
|
| 332 |
-
brand = vqa_brand
|
| 333 |
-
if brand and brand.lower() not in main.lower():
|
| 334 |
-
parts.append(f'Published by {brand}')
|
| 335 |
-
|
| 336 |
-
prods = analysis['classification'].get('products',[])
|
| 337 |
-
if prods and prods[0][1]>0.4 and prods[0][0] in ['comic book','manga','book']:
|
| 338 |
-
main = f'This is a {prods[0][0]} cover. ' + main
|
| 339 |
-
|
| 340 |
-
mat = analysis['vqa'].get('what material does this product appear to be made of','')
|
| 341 |
-
if mat and mat.lower() not in ['paper','unknown']:
|
| 342 |
-
parts.append(f'The material appears to be {mat}')
|
| 343 |
-
|
| 344 |
-
final = main
|
| 345 |
for p in parts:
|
| 346 |
final += " " + p + "."
|
| 347 |
-
final = re.sub(r'\s+',' ',final).strip()
|
| 348 |
-
if not final.endswith('.'):
|
|
|
|
| 349 |
return final
|
| 350 |
|
| 351 |
# === Initialize embeddings ===
|
|
@@ -353,15 +263,15 @@ print("Precomputing embeddings...")
|
|
| 353 |
precompute_embeddings()
|
| 354 |
print("Embeddings ready!")
|
| 355 |
|
| 356 |
-
# ===
|
| 357 |
def handle_single_image(file):
|
| 358 |
try:
|
| 359 |
img = Image.open(file.name).convert("RGB")
|
| 360 |
except Exception as e:
|
| 361 |
return f"Failed to open image: {e}"
|
| 362 |
try:
|
| 363 |
-
|
| 364 |
-
return build_narrative_alt_text(
|
| 365 |
except Exception as e:
|
| 366 |
return f"Analysis failed: {e}"
|
| 367 |
|
|
@@ -381,18 +291,17 @@ def process_images(files):
|
|
| 381 |
return "No images uploaded."
|
| 382 |
results = []
|
| 383 |
with ThreadPoolExecutor(max_workers=1) as exe:
|
| 384 |
-
futures = {exe.submit(handle_single_image,f): i for i,f in enumerate(files[:8])}
|
| 385 |
for fut in as_completed(futures):
|
| 386 |
idx = futures[fut]
|
| 387 |
try:
|
| 388 |
results.append(f"**Image {idx+1}:**\n{fut.result()}\n")
|
| 389 |
except Exception as e:
|
| 390 |
results.append(f"**Image {idx+1}:** Failed - {e}\n")
|
| 391 |
-
return "\n".join(sorted(results, key=lambda s:int(re.search(r"Image (\d+):",s).group(1))))
|
| 392 |
|
| 393 |
-
# === Gradio UI ===
|
| 394 |
with gr.Blocks(
|
| 395 |
-
title="🔍
|
| 396 |
theme=gr.themes.Soft(),
|
| 397 |
css="""
|
| 398 |
.gradio-container { max-width:1400px!important; }
|
|
@@ -400,52 +309,27 @@ with gr.Blocks(
|
|
| 400 |
.upload-section { border:2px dashed #ccc; border-radius:10px; padding:20px; }
|
| 401 |
"""
|
| 402 |
) as demo:
|
| 403 |
-
|
| 404 |
gr.Markdown("""
|
| 405 |
-
# 🖼️
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
-
|
| 409 |
-
-
|
| 410 |
-
-
|
| 411 |
-
- ✅ **Text integration**
|
| 412 |
-
- ✅ **Artistic context**
|
| 413 |
-
- ✅ **Enhanced BLIP prompting**
|
| 414 |
-
|
| 415 |
-
Upload images to generate professional-quality descriptions.
|
| 416 |
""")
|
| 417 |
|
| 418 |
with gr.Row():
|
| 419 |
with gr.Column(scale=1, elem_classes=["upload-section"]):
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
file_types=["image"],
|
| 423 |
-
height=200)
|
| 424 |
-
generate_btn = gr.Button("🚀 Generate Detailed Alt-Text", variant="primary", size="lg")
|
| 425 |
-
gr.Markdown("""
|
| 426 |
-
**Supported:** JPG, PNG, WebP, BMP
|
| 427 |
-
**Max:** 8 images/batch
|
| 428 |
-
**Tip:** Use clear, well-lit photos with visible text/branding.
|
| 429 |
-
""")
|
| 430 |
-
|
| 431 |
with gr.Column(scale=2):
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
gr.Markdown("""---
|
| 441 |
-
**🔧 Technical Features:**
|
| 442 |
-
- Enhanced BLIP & detailed prompts
|
| 443 |
-
- Comprehensive CLIP zero-shot
|
| 444 |
-
- Smart OCR preprocessing
|
| 445 |
-
- Color clustering & naming
|
| 446 |
-
- Optional VQA insights
|
| 447 |
-
- Robust error handling
|
| 448 |
-
""")
|
| 449 |
|
| 450 |
if __name__ == "__main__":
|
| 451 |
-
demo.launch(server_name="0.0.0.0", server_port=7860, share=False
|
|
|
|
| 6 |
import cv2
|
| 7 |
import numpy as np
|
| 8 |
import re
|
|
|
|
|
|
|
| 9 |
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 10 |
from sklearn.cluster import KMeans
|
| 11 |
+
from transformers import (
|
| 12 |
+
BlipProcessor,
|
| 13 |
+
BlipForConditionalGeneration,
|
| 14 |
+
BlipForQuestionAnswering
|
| 15 |
+
)
|
| 16 |
|
| 17 |
# === Device setup ===
|
| 18 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 19 |
torch.backends.cudnn.benchmark = True
|
|
|
|
| 20 |
print(f"Using device: {device}")
|
| 21 |
|
| 22 |
# === Model initialization ===
|
|
|
|
| 25 |
blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
|
| 26 |
blip_model.to(device).eval()
|
| 27 |
|
| 28 |
+
# BLIP for VQA (optional)
|
| 29 |
try:
|
| 30 |
blip_vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
|
| 31 |
blip_vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
|
|
|
|
| 46 |
if device.type == "cuda":
|
| 47 |
clip_model = clip_model.half()
|
| 48 |
|
| 49 |
+
# OCR reader
|
| 50 |
ocr_reader = easyocr.Reader(["en", "es", "fr", "de", "it"], gpu=(device.type == "cuda"))
|
| 51 |
|
| 52 |
+
# === Label sets (unchanged) ===
|
| 53 |
+
COMPREHENSIVE_PRODUCT_LABELS = [ … ] # same as before
|
| 54 |
+
COMPREHENSIVE_BRAND_LABELS = [ … ] # same as before
|
| 55 |
+
CHARACTER_LABELS = [ … ]
|
| 56 |
+
DETAIL_LABELS = [ … ]
|
| 57 |
+
|
| 58 |
+
# === Basic color palette for mapping ===
|
| 59 |
+
BASE_COLORS = {
|
| 60 |
+
'red': (255, 0, 0),
|
| 61 |
+
'green': (0, 128, 0),
|
| 62 |
+
'blue': (0, 0, 255),
|
| 63 |
+
'yellow': (255, 255, 0),
|
| 64 |
+
'orange': (255, 165, 0),
|
| 65 |
+
'purple': (128, 0, 128),
|
| 66 |
+
'pink': (255, 192, 203),
|
| 67 |
+
'brown': (165, 42, 42),
|
| 68 |
+
'gray': (128, 128, 128),
|
| 69 |
+
'black': (0, 0, 0),
|
| 70 |
+
'white': (255, 255, 255),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
}
|
| 72 |
|
| 73 |
+
def get_basic_color_name(rgb_tuple):
|
| 74 |
+
"""Map any RGB to nearest basic color name."""
|
| 75 |
+
best, best_name = float('inf'), 'unknown'
|
| 76 |
+
for name, ref in BASE_COLORS.items():
|
| 77 |
+
dist = sum((a - b) ** 2 for a, b in zip(rgb_tuple, ref)) ** 0.5
|
| 78 |
+
if dist < best:
|
| 79 |
+
best, best_name = dist, name
|
| 80 |
+
return best_name
|
| 81 |
|
| 82 |
def extract_dominant_colors(image: Image.Image, n_colors=3):
|
| 83 |
+
"""Use k-means on downsampled pixels, then map to basic colors."""
|
| 84 |
+
arr = np.array(image.resize((150, 150))).reshape(-1, 3)
|
| 85 |
+
# drop pure black/white
|
| 86 |
+
mask = ~((arr == [255,255,255]).all(axis=1) | (arr == [0,0,0]).all(axis=1))
|
| 87 |
+
arr = arr[mask] if mask.sum() > 0 else arr
|
| 88 |
+
if len(arr) < n_colors:
|
| 89 |
+
return []
|
| 90 |
kmeans = KMeans(n_clusters=n_colors, random_state=42, n_init=10).fit(arr)
|
| 91 |
colors = []
|
| 92 |
for center in kmeans.cluster_centers_:
|
| 93 |
rgb = tuple(map(int, center))
|
| 94 |
+
colors.append(get_basic_color_name(rgb))
|
| 95 |
+
# dedupe and preserve order
|
| 96 |
+
seen = set()
|
| 97 |
+
result = []
|
| 98 |
+
for c in colors:
|
| 99 |
+
if c not in seen:
|
| 100 |
+
seen.add(c)
|
| 101 |
+
result.append(c)
|
| 102 |
+
return result
|
| 103 |
+
|
| 104 |
+
# === Zero-shot classification setup ===
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
def precompute_embeddings():
|
| 106 |
+
global _prod_emb, _brand_emb, _char_emb, _detail_emb
|
| 107 |
+
prompts_prod = [f"a photo of a {l}" for l in COMPREHENSIVE_PRODUCT_LABELS]
|
| 108 |
+
prompts_brand = [f"a {b} product" for b in COMPREHENSIVE_BRAND_LABELS]
|
| 109 |
+
prompts_char = [f"a photo of a {c}" for c in CHARACTER_LABELS]
|
| 110 |
+
prompts_det = [f"an image showing {d}" for d in DETAIL_LABELS]
|
| 111 |
with torch.inference_mode():
|
| 112 |
+
_prod_emb = clip_model.encode_text(open_clip.tokenize(prompts_prod).to(device))
|
| 113 |
+
_prod_emb /= _prod_emb.norm(dim=-1, keepdim=True)
|
| 114 |
+
_brand_emb = clip_model.encode_text(open_clip.tokenize(prompts_brand).to(device))
|
| 115 |
+
_brand_emb /= _brand_emb.norm(dim=-1, keepdim=True)
|
| 116 |
+
_char_emb = clip_model.encode_text(open_clip.tokenize(prompts_char).to(device))
|
| 117 |
+
_char_emb /= _char_emb.norm(dim=-1, keepdim=True)
|
| 118 |
+
_det_emb = clip_model.encode_text(open_clip.tokenize(prompts_det).to(device))
|
| 119 |
+
_det_emb /= _det_emb.norm(dim=-1, keepdim=True)
|
| 120 |
|
| 121 |
def enhanced_zero_shot_classification(image: Image.Image):
|
| 122 |
with torch.inference_mode():
|
| 123 |
img_t = clip_preprocess(image).unsqueeze(0).to(device)
|
| 124 |
+
img_e = clip_model.encode_image(img_t)
|
| 125 |
+
img_e /= img_e.norm(dim=-1, keepdim=True)
|
| 126 |
+
results = {}
|
| 127 |
+
for labels, emb, key, topk in [
|
| 128 |
+
(COMPREHENSIVE_PRODUCT_LABELS, _prod_emb, 'products', 3),
|
| 129 |
+
(COMPREHENSIVE_BRAND_LABELS, _brand_emb, 'brands', 3),
|
| 130 |
+
(CHARACTER_LABELS, _char_emb, 'characters', 3),
|
| 131 |
+
(DETAIL_LABELS, _det_emb, 'details', 5),
|
| 132 |
]:
|
| 133 |
+
logits = (100.0 * img_e @ emb.T)
|
| 134 |
probs = torch.softmax(logits, dim=-1)
|
| 135 |
top = torch.topk(probs, topk, dim=-1)
|
| 136 |
+
results[key] = [
|
| 137 |
+
(labels[idx], float(prob))
|
| 138 |
+
for prob, idx in zip(top.values[0], top.indices[0])
|
| 139 |
+
]
|
| 140 |
+
return results
|
| 141 |
+
|
| 142 |
+
# === BLIP captioning with enhanced prompts ===
|
| 143 |
+
DETAILED_PROMPTS = [
|
| 144 |
+
"Provide a detailed, comprehensive description of this image including all visual elements, characters, objects, text, colors, setting, and artistic style.",
|
| 145 |
+
"Describe this image in great detail, mentioning the title, characters' appearance, clothing, expressions, objects they're holding, background elements, and any text visible.",
|
| 146 |
+
"Write a thorough description of this cover art or product image, including character details, clothing, facial expressions, objects, background setting, colors, and any visible text or branding.",
|
| 147 |
+
"Analyze this image completely: describe the main subject, their appearance and clothing, any objects or accessories, the background setting, color scheme, artistic style, and all visible text."
|
| 148 |
+
]
|
| 149 |
|
|
|
|
| 150 |
def get_enhanced_caption(image: Image.Image):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
captions = []
|
| 152 |
+
# unconditional baseline
|
| 153 |
try:
|
| 154 |
inputs = blip_processor(image, return_tensors="pt").to(device)
|
| 155 |
+
out = blip_model.generate(
|
| 156 |
+
**inputs, max_new_tokens=150, num_beams=8,
|
| 157 |
+
do_sample=True, temperature=0.8, top_p=0.9,
|
| 158 |
+
no_repeat_ngram_size=2, repetition_penalty=1.1
|
| 159 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
captions.append(blip_processor.decode(out[0], skip_special_tokens=True).strip())
|
| 161 |
except Exception as e:
|
| 162 |
+
print(f"Baseline caption error: {e}")
|
| 163 |
|
| 164 |
+
# detailed prompts
|
| 165 |
+
for prompt in DETAILED_PROMPTS:
|
| 166 |
try:
|
| 167 |
inputs = blip_processor(image, text=prompt, return_tensors="pt").to(device)
|
| 168 |
+
out = blip_model.generate(
|
| 169 |
+
**inputs, max_new_tokens=200, num_beams=8,
|
| 170 |
+
do_sample=True, temperature=0.7, top_p=0.85,
|
| 171 |
+
no_repeat_ngram_size=3, repetition_penalty=1.15
|
| 172 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
captions.append(blip_processor.decode(out[0], skip_special_tokens=True).strip())
|
| 174 |
except Exception as e:
|
| 175 |
+
print(f"Prompt caption error: {e}")
|
| 176 |
+
|
| 177 |
+
if not captions:
|
| 178 |
+
return "Unable to generate a caption."
|
| 179 |
|
| 180 |
+
# pick the most content-rich (longest) caption
|
| 181 |
+
best = max(captions, key=lambda s: len(s.split()))
|
| 182 |
+
best = re.sub(r'^(a|an|the)\s+', '', best, flags=re.IGNORECASE).strip().capitalize()
|
| 183 |
+
return best if best.endswith('.') else best + '.'
|
| 184 |
+
|
| 185 |
+
# === OCR ===
|
| 186 |
+
def extract_comprehensive_ocr(image: Image.Image):
|
| 187 |
+
try:
|
| 188 |
+
arr = np.array(image)
|
| 189 |
+
gray = cv2.cvtColor(arr, cv2.COLOR_RGB2GRAY)
|
| 190 |
+
preproc = [arr]
|
| 191 |
+
clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8)).apply(gray)
|
| 192 |
+
preproc.append(cv2.cvtColor(clahe, cv2.COLOR_GRAY2RGB))
|
| 193 |
+
_, th = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
|
| 194 |
+
preproc.append(cv2.cvtColor(th, cv2.COLOR_GRAY2RGB))
|
| 195 |
+
|
| 196 |
+
texts = []
|
| 197 |
+
for p in preproc:
|
| 198 |
+
for _, txt, conf in ocr_reader.readtext(p, detail=1, paragraph=True):
|
| 199 |
+
if conf > 0.3 and len(txt.strip()) > 1:
|
| 200 |
+
texts.append((txt.strip(), conf))
|
| 201 |
+
# dedupe
|
| 202 |
+
seen = set()
|
| 203 |
+
unique = []
|
| 204 |
+
for txt, conf in sorted(texts, key=lambda x: x[1], reverse=True):
|
| 205 |
+
key = re.sub(r'\W+', '', txt.lower())
|
| 206 |
+
if key not in seen:
|
| 207 |
+
seen.add(key)
|
| 208 |
+
unique.append(txt)
|
| 209 |
+
return " | ".join(unique[:5])
|
| 210 |
+
except Exception as e:
|
| 211 |
+
print(f"OCR error: {e}")
|
| 212 |
+
return ""
|
| 213 |
|
| 214 |
# === Comprehensive pipeline ===
|
| 215 |
def comprehensive_image_analysis(image: Image.Image):
|
| 216 |
+
# resize for efficiency
|
| 217 |
if max(image.size) > 1024:
|
| 218 |
image = image.copy()
|
| 219 |
image.thumbnail((1024,1024), Image.Resampling.LANCZOS)
|
| 220 |
|
| 221 |
+
return {
|
| 222 |
'caption': get_enhanced_caption(image),
|
|
|
|
| 223 |
'classification': enhanced_zero_shot_classification(image),
|
| 224 |
'ocr_text': extract_comprehensive_ocr(image),
|
| 225 |
'colors': extract_dominant_colors(image)
|
| 226 |
}
|
|
|
|
| 227 |
|
| 228 |
+
# === Build narrative alt-text ===
|
| 229 |
def build_narrative_alt_text(analysis):
|
| 230 |
+
main_desc = analysis['caption']
|
| 231 |
parts = []
|
| 232 |
|
| 233 |
+
# OCR text
|
| 234 |
+
ocr = analysis['ocr_text']
|
| 235 |
if ocr:
|
| 236 |
+
parts.append(f'Visible text: "{ocr}"')
|
| 237 |
+
|
| 238 |
+
# Dominant colors
|
| 239 |
+
cols = analysis['colors']
|
| 240 |
+
if cols:
|
| 241 |
+
if len(cols) == 1:
|
| 242 |
+
parts.append(f"The primary color is {cols[0]}")
|
| 243 |
else:
|
| 244 |
+
col_str = ", ".join(cols[:-1]) + f", and {cols[-1]}"
|
| 245 |
+
parts.append(f"The color palette features {col_str}")
|
| 246 |
|
| 247 |
+
# Brand (only if high confidence)
|
| 248 |
+
brands = analysis['classification']['brands']
|
| 249 |
+
if brands and brands[0][1] > 0.6:
|
| 250 |
+
parts.append(f"Published by {brands[0][0]}")
|
| 251 |
|
| 252 |
+
# Combine
|
| 253 |
+
final = main_desc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 254 |
for p in parts:
|
| 255 |
final += " " + p + "."
|
| 256 |
+
final = re.sub(r'\s+', ' ', final).strip()
|
| 257 |
+
if not final.endswith('.'):
|
| 258 |
+
final += '.'
|
| 259 |
return final
|
| 260 |
|
| 261 |
# === Initialize embeddings ===
|
|
|
|
| 263 |
precompute_embeddings()
|
| 264 |
print("Embeddings ready!")
|
| 265 |
|
| 266 |
+
# === Gradio interface ===
|
| 267 |
def handle_single_image(file):
|
| 268 |
try:
|
| 269 |
img = Image.open(file.name).convert("RGB")
|
| 270 |
except Exception as e:
|
| 271 |
return f"Failed to open image: {e}"
|
| 272 |
try:
|
| 273 |
+
analysis = comprehensive_image_analysis(img)
|
| 274 |
+
return build_narrative_alt_text(analysis)
|
| 275 |
except Exception as e:
|
| 276 |
return f"Analysis failed: {e}"
|
| 277 |
|
|
|
|
| 291 |
return "No images uploaded."
|
| 292 |
results = []
|
| 293 |
with ThreadPoolExecutor(max_workers=1) as exe:
|
| 294 |
+
futures = {exe.submit(handle_single_image, f): i for i, f in enumerate(files[:8])}
|
| 295 |
for fut in as_completed(futures):
|
| 296 |
idx = futures[fut]
|
| 297 |
try:
|
| 298 |
results.append(f"**Image {idx+1}:**\n{fut.result()}\n")
|
| 299 |
except Exception as e:
|
| 300 |
results.append(f"**Image {idx+1}:** Failed - {e}\n")
|
| 301 |
+
return "\n".join(sorted(results, key=lambda s: int(re.search(r"Image (\d+):", s).group(1))))
|
| 302 |
|
|
|
|
| 303 |
with gr.Blocks(
|
| 304 |
+
title="🔍 Optimized Narrative Alt-Text Generator",
|
| 305 |
theme=gr.themes.Soft(),
|
| 306 |
css="""
|
| 307 |
.gradio-container { max-width:1400px!important; }
|
|
|
|
| 309 |
.upload-section { border:2px dashed #ccc; border-radius:10px; padding:20px; }
|
| 310 |
"""
|
| 311 |
) as demo:
|
|
|
|
| 312 |
gr.Markdown("""
|
| 313 |
+
# 🖼️ Optimized Alt-Text Generator
|
| 314 |
+
|
| 315 |
+
- Improved basic color mapping
|
| 316 |
+
- Brand only if high confidence (≥60%)
|
| 317 |
+
- Removed unreliable VQA-brand/material
|
| 318 |
+
- K-means → basic colors for clearer palettes
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 319 |
""")
|
| 320 |
|
| 321 |
with gr.Row():
|
| 322 |
with gr.Column(scale=1, elem_classes=["upload-section"]):
|
| 323 |
+
inp = gr.File(label="Upload Images", file_count="multiple", file_types=["image"], height=200)
|
| 324 |
+
btn = gr.Button("Generate Alt-Text", variant="primary", size="lg")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
with gr.Column(scale=2):
|
| 326 |
+
gal = gr.Gallery(label="Preview", columns=4, rows=2, height=350, object_fit="contain")
|
| 327 |
+
|
| 328 |
+
out = gr.Textbox(label="Generated Alt-Text", lines=20, show_copy_button=True,
|
| 329 |
+
elem_classes=["description-output"], placeholder="...")
|
| 330 |
+
|
| 331 |
+
inp.change(preview_images_grid, inp, gal, show_progress="hidden")
|
| 332 |
+
btn.click(process_images, inp, out, show_progress=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 333 |
|
| 334 |
if __name__ == "__main__":
|
| 335 |
+
demo.launch(server_name="0.0.0.0", server_port=7860, share=False)
|