Update app.py
Browse files
app.py
CHANGED
|
@@ -54,42 +54,33 @@ COMPREHENSIVE_PRODUCT_LABELS = [
|
|
| 54 |
"trucker hat", "baseball cap", "beanie", "cap", "hat", "sunglasses", "watch", "jewelry",
|
| 55 |
"necklace", "bracelet", "ring", "earrings", "bag", "backpack", "purse", "wallet",
|
| 56 |
"belt", "scarf", "gloves", "shoes", "sneakers", "boots", "sandals", "heels",
|
| 57 |
-
|
| 58 |
# Electronics & Tech
|
| 59 |
"smartphone", "laptop", "tablet", "headphones", "earbuds", "speaker", "camera",
|
| 60 |
"gaming console", "controller", "keyboard", "mouse", "monitor", "TV", "remote",
|
| 61 |
"charger", "cable", "smartwatch", "fitness tracker", "drone", "robot",
|
| 62 |
-
|
| 63 |
# Toys & Games
|
| 64 |
"board game", "card game", "puzzle", "action figure", "doll", "toy car", "LEGO",
|
| 65 |
"building blocks", "stuffed animal", "teddy bear", "toy robot", "game controller",
|
| 66 |
"dice", "playing cards", "collectible card", "miniature figure", "model kit",
|
| 67 |
-
|
| 68 |
# Beauty & Personal Care
|
| 69 |
"perfume bottle", "cologne", "lipstick", "mascara", "foundation", "powder compact",
|
| 70 |
"nail polish", "skincare bottle", "lotion", "cream jar", "soap", "shampoo",
|
| 71 |
"conditioner", "toothbrush", "razor", "makeup brush", "mirror", "hair dryer",
|
| 72 |
-
|
| 73 |
# Food & Beverages
|
| 74 |
"water bottle", "soda can", "beer bottle", "wine bottle", "coffee mug", "tea cup",
|
| 75 |
"food package", "cereal box", "snack bag", "candy bar", "chocolate", "cookie package",
|
| 76 |
-
|
| 77 |
# Books & Media
|
| 78 |
"book", "magazine", "comic book", "DVD", "Blu-ray", "CD", "vinyl record",
|
| 79 |
"notebook", "journal", "planner", "calendar", "poster", "artwork", "photo frame",
|
| 80 |
-
|
| 81 |
# Sports & Fitness
|
| 82 |
"basketball", "football", "soccer ball", "tennis ball", "baseball", "golf ball",
|
| 83 |
"dumbbells", "yoga mat", "water bottle", "sports equipment", "helmet", "gloves",
|
| 84 |
-
|
| 85 |
# Home & Kitchen
|
| 86 |
"coffee maker", "blender", "toaster", "microwave", "pan", "pot", "plate", "bowl",
|
| 87 |
"cup", "glass", "utensils", "knife", "cutting board", "storage container",
|
| 88 |
-
|
| 89 |
# Art & Craft Supplies
|
| 90 |
"paint brush", "marker", "pen", "pencil", "crayon", "paint tube", "canvas",
|
| 91 |
"sketchbook", "ruler", "scissors", "glue", "tape", "stickers",
|
| 92 |
-
|
| 93 |
# Packaging & Containers
|
| 94 |
"box", "package", "container", "jar", "bottle", "can", "tube", "pouch", "bag"
|
| 95 |
]
|
|
@@ -98,27 +89,21 @@ COMPREHENSIVE_BRAND_LABELS = [
|
|
| 98 |
# Tech Brands
|
| 99 |
"Apple", "Samsung", "Google", "Microsoft", "Sony", "Nintendo", "PlayStation", "Xbox",
|
| 100 |
"HP", "Dell", "Lenovo", "ASUS", "Acer", "Canon", "Nikon", "GoPro", "DJI",
|
| 101 |
-
|
| 102 |
# Fashion & Accessories
|
| 103 |
"Nike", "Adidas", "Puma", "Under Armour", "Reebok", "Converse", "Vans", "New Balance",
|
| 104 |
"Ray-Ban", "Oakley", "Rolex", "Casio", "Fossil", "Michael Kors", "Coach", "Louis Vuitton",
|
| 105 |
-
|
| 106 |
# Beauty & Personal Care
|
| 107 |
"L'Oreal", "Maybelline", "MAC", "Sephora", "Clinique", "Estee Lauder", "Chanel",
|
| 108 |
"Dior", "YSL", "Urban Decay", "Too Faced", "NARS", "Fenty Beauty",
|
| 109 |
-
|
| 110 |
# Gaming & Toys
|
| 111 |
"Hasbro", "Mattel", "LEGO", "Ravensburger", "Fantasy Flight Games", "Asmodee",
|
| 112 |
"Days of Wonder", "CMON", "Wizards of the Coast", "PokΓ©mon", "Yu-Gi-Oh",
|
| 113 |
-
|
| 114 |
# Food & Beverages
|
| 115 |
"Coca-Cola", "Pepsi", "Starbucks", "Red Bull", "Monster", "Gatorade", "Powerade",
|
| 116 |
"Nestle", "Hershey's", "Mars", "Ferrero", "Lindt", "Godiva",
|
| 117 |
-
|
| 118 |
# Art & Craft
|
| 119 |
"Crayola", "Sharpie", "Prismacolor", "Winsor & Newton", "Molotow", "Posca",
|
| 120 |
"Staedtler", "Faber-Castell", "Sakura", "Tombow", "Pilot", "Uni-ball",
|
| 121 |
-
|
| 122 |
# Specialty/Generic
|
| 123 |
"unknown", "generic", "no brand visible"
|
| 124 |
]
|
|
@@ -137,120 +122,43 @@ DETAIL_LABELS = [
|
|
| 137 |
]
|
| 138 |
|
| 139 |
# === Enhanced color detection ===
|
| 140 |
-
# Comprehensive color mapping (CSS3 colors)
|
| 141 |
CSS3_COLORS = {
|
| 142 |
-
|
| 143 |
-
(127, 255, 212): 'aquamarine', (240, 255, 255): 'azure', (245, 245, 220): 'beige',
|
| 144 |
-
(255, 228, 196): 'bisque', (0, 0, 0): 'black', (255, 235, 205): 'blanchedalmond',
|
| 145 |
-
(0, 0, 255): 'blue', (138, 43, 226): 'blueviolet', (165, 42, 42): 'brown',
|
| 146 |
-
(222, 184, 135): 'burlywood', (95, 158, 160): 'cadetblue', (127, 255, 0): 'chartreuse',
|
| 147 |
-
(210, 105, 30): 'chocolate', (255, 127, 80): 'coral', (100, 149, 237): 'cornflowerblue',
|
| 148 |
-
(255, 248, 220): 'cornsilk', (220, 20, 60): 'crimson', (0, 255, 255): 'cyan',
|
| 149 |
-
(0, 0, 139): 'darkblue', (0, 139, 139): 'darkcyan', (184, 134, 11): 'darkgoldenrod',
|
| 150 |
-
(169, 169, 169): 'darkgray', (0, 100, 0): 'darkgreen', (189, 183, 107): 'darkkhaki',
|
| 151 |
-
(139, 0, 139): 'darkmagenta', (85, 107, 47): 'darkolivegreen', (255, 140, 0): 'darkorange',
|
| 152 |
-
(153, 50, 204): 'darkorchid', (139, 0, 0): 'darkred', (233, 150, 122): 'darksalmon',
|
| 153 |
-
(143, 188, 143): 'darkseagreen', (72, 61, 139): 'darkslateblue', (47, 79, 79): 'darkslategray',
|
| 154 |
-
(0, 206, 209): 'darkturquoise', (148, 0, 211): 'darkviolet', (255, 20, 147): 'deeppink',
|
| 155 |
-
(0, 191, 255): 'deepskyblue', (105, 105, 105): 'dimgray', (30, 144, 255): 'dodgerblue',
|
| 156 |
-
(178, 34, 34): 'firebrick', (255, 250, 240): 'floralwhite', (34, 139, 34): 'forestgreen',
|
| 157 |
-
(255, 0, 255): 'fuchsia', (220, 220, 220): 'gainsboro', (248, 248, 255): 'ghostwhite',
|
| 158 |
-
(255, 215, 0): 'gold', (218, 165, 32): 'goldenrod', (128, 128, 128): 'gray',
|
| 159 |
-
(0, 128, 0): 'green', (173, 255, 47): 'greenyellow', (240, 255, 240): 'honeydew',
|
| 160 |
-
(255, 105, 180): 'hotpink', (205, 92, 92): 'indianred', (75, 0, 130): 'indigo',
|
| 161 |
-
(255, 255, 240): 'ivory', (240, 230, 140): 'khaki', (230, 230, 250): 'lavender',
|
| 162 |
-
(255, 240, 245): 'lavenderblush', (124, 252, 0): 'lawngreen', (255, 250, 205): 'lemonchiffon',
|
| 163 |
-
(173, 216, 230): 'lightblue', (240, 128, 128): 'lightcoral', (224, 255, 255): 'lightcyan',
|
| 164 |
-
(250, 250, 210): 'lightgoldenrodyellow', (211, 211, 211): 'lightgray', (144, 238, 144): 'lightgreen',
|
| 165 |
-
(255, 182, 193): 'lightpink', (255, 160, 122): 'lightsalmon', (32, 178, 170): 'lightseagreen',
|
| 166 |
-
(135, 206, 250): 'lightskyblue', (119, 136, 153): 'lightslategray', (176, 196, 222): 'lightsteelblue',
|
| 167 |
-
(255, 255, 224): 'lightyellow', (0, 255, 0): 'lime', (50, 205, 50): 'limegreen',
|
| 168 |
-
(250, 240, 230): 'linen', (255, 0, 255): 'magenta', (128, 0, 0): 'maroon',
|
| 169 |
-
(102, 205, 170): 'mediumaquamarine', (0, 0, 205): 'mediumblue', (186, 85, 211): 'mediumorchid',
|
| 170 |
-
(147, 112, 219): 'mediumpurple', (60, 179, 113): 'mediumseagreen', (123, 104, 238): 'mediumslateblue',
|
| 171 |
-
(0, 250, 154): 'mediumspringgreen', (72, 209, 204): 'mediumturquoise', (199, 21, 133): 'mediumvioletred',
|
| 172 |
-
(25, 25, 112): 'midnightblue', (245, 255, 250): 'mintcream', (255, 228, 225): 'mistyrose',
|
| 173 |
-
(255, 228, 181): 'moccasin', (255, 222, 173): 'navajowhite', (0, 0, 128): 'navy',
|
| 174 |
-
(253, 245, 230): 'oldlace', (128, 128, 0): 'olive', (107, 142, 35): 'olivedrab',
|
| 175 |
-
(255, 165, 0): 'orange', (255, 69, 0): 'orangered', (218, 112, 214): 'orchid',
|
| 176 |
-
(238, 232, 170): 'palegoldenrod', (152, 251, 152): 'palegreen', (175, 238, 238): 'paleturquoise',
|
| 177 |
-
(219, 112, 147): 'palevioletred', (255, 239, 213): 'papayawhip', (255, 218, 185): 'peachpuff',
|
| 178 |
-
(205, 133, 63): 'peru', (255, 192, 203): 'pink', (221, 160, 221): 'plum',
|
| 179 |
-
(176, 224, 230): 'powderblue', (128, 0, 128): 'purple', (255, 0, 0): 'red',
|
| 180 |
-
(188, 143, 143): 'rosybrown', (65, 105, 225): 'royalblue', (139, 69, 19): 'saddlebrown',
|
| 181 |
-
(250, 128, 114): 'salmon', (244, 164, 96): 'sandybrown', (46, 139, 87): 'seagreen',
|
| 182 |
-
(255, 245, 238): 'seashell', (160, 82, 45): 'sienna', (192, 192, 192): 'silver',
|
| 183 |
-
(135, 206, 235): 'skyblue', (106, 90, 205): 'slateblue', (112, 128, 144): 'slategray',
|
| 184 |
-
(255, 250, 250): 'snow', (0, 255, 127): 'springgreen', (70, 130, 180): 'steelblue',
|
| 185 |
-
(210, 180, 140): 'tan', (0, 128, 128): 'teal', (216, 191, 216): 'thistle',
|
| 186 |
-
(255, 99, 71): 'tomato', (64, 224, 208): 'turquoise', (238, 130, 238): 'violet',
|
| 187 |
-
(245, 222, 179): 'wheat', (255, 255, 255): 'white', (245, 245, 245): 'whitesmoke',
|
| 188 |
-
(255, 255, 0): 'yellow', (154, 205, 50): 'yellowgreen'
|
| 189 |
}
|
| 190 |
|
| 191 |
def get_color_name(rgb_tuple):
|
| 192 |
-
"""Find the closest named color to the given RGB tuple."""
|
| 193 |
min_distance = float('inf')
|
| 194 |
closest_color = 'unknown'
|
| 195 |
-
|
| 196 |
for color_rgb, color_name in CSS3_COLORS.items():
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
min_distance = distance
|
| 201 |
-
closest_color = color_name
|
| 202 |
-
|
| 203 |
return closest_color
|
| 204 |
|
| 205 |
def extract_dominant_colors(image: Image.Image, n_colors=3):
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
mask = ~((img_array == [255, 255, 255]).all(axis=1) | (img_array == [0, 0, 0]).all(axis=1))
|
| 212 |
-
if mask.sum() > 0:
|
| 213 |
-
img_array = img_array[mask]
|
| 214 |
-
|
| 215 |
-
if len(img_array) < n_colors:
|
| 216 |
-
return []
|
| 217 |
-
|
| 218 |
-
kmeans = KMeans(n_clusters=n_colors, random_state=42, n_init=10)
|
| 219 |
-
kmeans.fit(img_array)
|
| 220 |
-
|
| 221 |
colors = []
|
| 222 |
for center in kmeans.cluster_centers_:
|
| 223 |
rgb = tuple(map(int, center))
|
| 224 |
-
|
| 225 |
-
color_name = get_color_name(rgb)
|
| 226 |
-
colors.append(color_name)
|
| 227 |
-
except:
|
| 228 |
-
colors.append(f"RGB{rgb}")
|
| 229 |
-
|
| 230 |
return colors
|
| 231 |
|
| 232 |
-
# === Enhanced VQA for detailed analysis ===
|
| 233 |
def ask_visual_question(image: Image.Image, question: str):
|
| 234 |
-
|
| 235 |
-
if not VQA_AVAILABLE:
|
| 236 |
-
return ""
|
| 237 |
-
|
| 238 |
try:
|
| 239 |
inputs = blip_vqa_processor(image, question, return_tensors="pt").to(device)
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
return answer.strip()
|
| 244 |
-
except Exception as e:
|
| 245 |
-
print(f"VQA error: {e}")
|
| 246 |
return ""
|
| 247 |
|
| 248 |
def analyze_with_vqa(image: Image.Image):
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
return {}
|
| 252 |
-
|
| 253 |
-
questions = [
|
| 254 |
"What type of product is this?",
|
| 255 |
"What brand is visible in this image?",
|
| 256 |
"What colors are prominent in this image?",
|
|
@@ -262,149 +170,87 @@ def analyze_with_vqa(image: Image.Image):
|
|
| 262 |
"Are there any logos or symbols visible?",
|
| 263 |
"What is the shape of the main object?"
|
| 264 |
]
|
| 265 |
-
|
| 266 |
results = {}
|
| 267 |
-
for
|
| 268 |
-
|
| 269 |
-
if
|
| 270 |
-
key =
|
| 271 |
-
results[key] =
|
| 272 |
-
|
| 273 |
return results
|
| 274 |
|
| 275 |
-
# === Enhanced OCR with better text processing ===
|
| 276 |
def extract_comprehensive_ocr(image: Image.Image):
|
| 277 |
-
"""Enhanced OCR with better preprocessing and post-processing."""
|
| 278 |
try:
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
all_texts = []
|
| 301 |
-
|
| 302 |
-
for proc_img in preprocessed_images:
|
| 303 |
-
try:
|
| 304 |
-
results = ocr_reader.readtext(proc_img, detail=1, paragraph=True)
|
| 305 |
-
for (bbox, text, confidence) in results:
|
| 306 |
-
if confidence > 0.3 and len(text.strip()) > 1:
|
| 307 |
-
all_texts.append((text.strip(), confidence))
|
| 308 |
-
except:
|
| 309 |
-
continue
|
| 310 |
-
|
| 311 |
-
# Remove duplicates and low confidence texts
|
| 312 |
-
unique_texts = []
|
| 313 |
-
seen = set()
|
| 314 |
-
for text, conf in sorted(all_texts, key=lambda x: x[1], reverse=True):
|
| 315 |
-
text_clean = re.sub(r'\W+', '', text.lower())
|
| 316 |
-
if text_clean not in seen and len(text_clean) > 1:
|
| 317 |
-
unique_texts.append(text)
|
| 318 |
-
seen.add(text_clean)
|
| 319 |
-
|
| 320 |
-
return " | ".join(unique_texts[:5]) # Limit to top 5 unique texts
|
| 321 |
-
|
| 322 |
-
except Exception as e:
|
| 323 |
-
print(f"OCR error: {e}")
|
| 324 |
return ""
|
| 325 |
|
| 326 |
-
# ===
|
| 327 |
def precompute_embeddings():
|
| 328 |
-
"""Precompute embeddings for all label categories."""
|
| 329 |
global _product_embeddings, _brand_embeddings, _character_embeddings, _detail_embeddings
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
detail_prompts = [f"an image showing {label}" for label in DETAIL_LABELS]
|
| 335 |
-
|
| 336 |
with torch.inference_mode():
|
| 337 |
-
_product_embeddings
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
_brand_embeddings = _brand_embeddings / _brand_embeddings.norm(dim=-1, keepdim=True)
|
| 342 |
-
|
| 343 |
-
_character_embeddings = clip_model.encode_text(open_clip.tokenize(character_prompts).to(device))
|
| 344 |
-
_character_embeddings = _character_embeddings / _character_embeddings.norm(dim=-1, keepdim=True)
|
| 345 |
-
|
| 346 |
-
_detail_embeddings = clip_model.encode_text(open_clip.tokenize(detail_prompts).to(device))
|
| 347 |
-
_detail_embeddings = _detail_embeddings / _detail_embeddings.norm(dim=-1, keepdim=True)
|
| 348 |
|
| 349 |
def enhanced_zero_shot_classification(image: Image.Image):
|
| 350 |
-
"""Enhanced zero-shot classification across multiple categories."""
|
| 351 |
with torch.inference_mode():
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
results['brands'] = [(COMPREHENSIVE_BRAND_LABELS[idx], float(prob))
|
| 370 |
-
for prob, idx in zip(top_brands.values[0], top_brands.indices[0])]
|
| 371 |
-
|
| 372 |
-
# Character classification
|
| 373 |
-
character_logits = 100.0 * img_emb @ _character_embeddings.T
|
| 374 |
-
character_probs = torch.softmax(character_logits, dim=-1)
|
| 375 |
-
top_characters = torch.topk(character_probs, 3, dim=-1)
|
| 376 |
-
results['characters'] = [(CHARACTER_LABELS[idx], float(prob))
|
| 377 |
-
for prob, idx in zip(top_characters.values[0], top_characters.indices[0])]
|
| 378 |
-
|
| 379 |
-
# Detail classification
|
| 380 |
-
detail_logits = 100.0 * img_emb @ _detail_embeddings.T
|
| 381 |
-
detail_probs = torch.softmax(detail_logits, dim=-1)
|
| 382 |
-
top_details = torch.topk(detail_probs, 5, dim=-1)
|
| 383 |
-
results['details'] = [(DETAIL_LABELS[idx], float(prob))
|
| 384 |
-
for prob, idx in zip(top_details.values[0], top_details.indices[0])]
|
| 385 |
-
|
| 386 |
-
return results
|
| 387 |
-
|
| 388 |
-
# === Enhanced BLIP captioning with detailed prompts ===
|
| 389 |
def get_enhanced_caption(image: Image.Image):
|
| 390 |
-
"""Generate detailed narrative captions with specific prompts."""
|
| 391 |
detailed_prompts = [
|
| 392 |
"Provide a detailed, comprehensive description of this image including all visual elements, characters, objects, text, colors, setting, and artistic style.",
|
| 393 |
"Describe this image in great detail, mentioning the title, characters' appearance, clothing, expressions, objects they're holding, background elements, and any text visible.",
|
| 394 |
"Write a thorough description of this cover art or product image, including character details, clothing, facial expressions, objects, background setting, colors, and any visible text or branding.",
|
| 395 |
-
"Analyze this image completely: describe the main subject, their appearance and clothing, any objects or accessories, the background setting, color scheme, artistic style, and all visible text."
|
| 396 |
]
|
| 397 |
-
|
| 398 |
captions = []
|
| 399 |
-
|
| 400 |
-
# Try unconditional first for baseline
|
| 401 |
try:
|
| 402 |
inputs = blip_processor(image, return_tensors="pt").to(device)
|
| 403 |
with torch.inference_mode():
|
| 404 |
out = blip_model.generate(
|
| 405 |
**inputs,
|
| 406 |
-
max_new_tokens=150,
|
| 407 |
-
num_beams=
|
| 408 |
do_sample=True,
|
| 409 |
temperature=0.8,
|
| 410 |
top_p=0.9,
|
|
@@ -412,20 +258,19 @@ def get_enhanced_caption(image: Image.Image):
|
|
| 412 |
no_repeat_ngram_size=2,
|
| 413 |
repetition_penalty=1.1
|
| 414 |
)
|
| 415 |
-
|
| 416 |
-
captions.append(caption)
|
| 417 |
except Exception as e:
|
| 418 |
print(f"Unconditional caption error: {e}")
|
| 419 |
-
|
| 420 |
-
# Try detailed prompts
|
| 421 |
-
for prompt in detailed_prompts
|
| 422 |
try:
|
| 423 |
inputs = blip_processor(image, text=prompt, return_tensors="pt").to(device)
|
| 424 |
with torch.inference_mode():
|
| 425 |
out = blip_model.generate(
|
| 426 |
**inputs,
|
| 427 |
-
max_new_tokens=
|
| 428 |
-
num_beams=
|
| 429 |
do_sample=True,
|
| 430 |
temperature=0.7,
|
| 431 |
top_p=0.85,
|
|
@@ -433,298 +278,174 @@ def get_enhanced_caption(image: Image.Image):
|
|
| 433 |
no_repeat_ngram_size=3,
|
| 434 |
repetition_penalty=1.15
|
| 435 |
)
|
| 436 |
-
|
| 437 |
-
caption = blip_processor.decode(out[0], skip_special_tokens=True).strip()
|
| 438 |
-
|
| 439 |
-
# Filter out prompts that got repeated in output
|
| 440 |
-
if not any(phrase in caption.lower() for phrase in [
|
| 441 |
-
"provide a detailed", "describe this image", "write a thorough",
|
| 442 |
-
"analyze this image", "comprehensive description"
|
| 443 |
-
]):
|
| 444 |
-
captions.append(caption)
|
| 445 |
-
|
| 446 |
except Exception as e:
|
| 447 |
-
print(f"Detailed caption error: {e}")
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
# Return the longest, most detailed caption
|
| 451 |
if captions:
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
return "Unable to generate detailed caption"
|
| 458 |
|
| 459 |
-
# === Comprehensive
|
| 460 |
def comprehensive_image_analysis(image: Image.Image):
|
| 461 |
-
"""Main analysis pipeline combining all techniques."""
|
| 462 |
-
# Resize for efficiency
|
| 463 |
if max(image.size) > 1024:
|
| 464 |
image = image.copy()
|
| 465 |
-
image.thumbnail((1024,
|
| 466 |
-
|
| 467 |
-
analysis = {
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
analysis['vqa'] = analyze_with_vqa(image)
|
| 475 |
-
else:
|
| 476 |
-
analysis['vqa'] = {}
|
| 477 |
-
|
| 478 |
-
# Zero-shot classification
|
| 479 |
-
analysis['classification'] = enhanced_zero_shot_classification(image)
|
| 480 |
-
|
| 481 |
-
# OCR analysis
|
| 482 |
-
analysis['ocr_text'] = extract_comprehensive_ocr(image)
|
| 483 |
-
|
| 484 |
-
# Color analysis
|
| 485 |
-
analysis['colors'] = extract_dominant_colors(image)
|
| 486 |
-
|
| 487 |
return analysis
|
| 488 |
|
| 489 |
-
# === Narrative alt-text generation ===
|
| 490 |
def build_narrative_alt_text(analysis):
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
# Add OCR text information naturally
|
| 500 |
-
ocr_text = analysis.get('ocr_text', '')
|
| 501 |
-
if ocr_text and len(ocr_text.strip()) > 2:
|
| 502 |
-
# Try to integrate OCR text naturally into description
|
| 503 |
-
if 'vol' in ocr_text.lower() or 'volume' in ocr_text.lower():
|
| 504 |
-
enhancement_parts.append(f"Text visible includes: \"{ocr_text}\"")
|
| 505 |
-
elif any(brand in ocr_text.lower() for brand in ['manga', 'comic', 'book']):
|
| 506 |
-
enhancement_parts.append(f"The title text reads: \"{ocr_text}\"")
|
| 507 |
else:
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
product_type = products[0][0]
|
| 544 |
-
if product_type in ['comic book', 'manga', 'book'] and product_type not in main_description.lower():
|
| 545 |
-
main_description = f"This is a {product_type} cover. " + main_description
|
| 546 |
-
|
| 547 |
-
# Add VQA insights that provide additional context
|
| 548 |
-
vqa = analysis.get('vqa', {})
|
| 549 |
-
material = vqa.get('material does this product appear to be made of', '')
|
| 550 |
-
if material and len(material) > 3 and material.lower() not in ['paper', 'unknown']:
|
| 551 |
-
enhancement_parts.append(f"The material appears to be {material}")
|
| 552 |
-
|
| 553 |
-
# Combine main description with enhancements
|
| 554 |
-
if enhancement_parts:
|
| 555 |
-
# Add enhancements naturally to the main description
|
| 556 |
-
enhanced_description = main_description
|
| 557 |
-
for enhancement in enhancement_parts:
|
| 558 |
-
enhanced_description += f" {enhancement}."
|
| 559 |
-
else:
|
| 560 |
-
enhanced_description = main_description
|
| 561 |
-
|
| 562 |
-
# Clean up the final description
|
| 563 |
-
enhanced_description = re.sub(r'\s+', ' ', enhanced_description) # Remove extra spaces
|
| 564 |
-
enhanced_description = re.sub(r'\.+', '.', enhanced_description) # Remove multiple periods
|
| 565 |
-
enhanced_description = enhanced_description.strip()
|
| 566 |
-
|
| 567 |
-
# Ensure proper ending
|
| 568 |
-
if not enhanced_description.endswith('.'):
|
| 569 |
-
enhanced_description += '.'
|
| 570 |
-
|
| 571 |
-
return enhanced_description
|
| 572 |
|
| 573 |
# === Initialize embeddings ===
|
| 574 |
print("Precomputing embeddings...")
|
| 575 |
precompute_embeddings()
|
| 576 |
print("Embeddings ready!")
|
| 577 |
|
| 578 |
-
# ===
|
| 579 |
def handle_single_image(file):
|
| 580 |
-
"""Process a single image file."""
|
| 581 |
try:
|
| 582 |
-
|
| 583 |
except Exception as e:
|
| 584 |
return f"Failed to open image: {e}"
|
| 585 |
-
|
| 586 |
try:
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
return alt_text
|
| 590 |
except Exception as e:
|
| 591 |
return f"Analysis failed: {e}"
|
| 592 |
|
| 593 |
def preview_images_grid(files):
|
| 594 |
-
"""Generate preview grid for uploaded images."""
|
| 595 |
-
if not files:
|
| 596 |
-
return []
|
| 597 |
previews = []
|
| 598 |
-
for
|
| 599 |
try:
|
| 600 |
-
img = Image.open(
|
| 601 |
-
img.thumbnail((200,
|
| 602 |
previews.append(img)
|
| 603 |
except:
|
| 604 |
continue
|
| 605 |
return previews
|
| 606 |
|
| 607 |
def process_images(files):
|
| 608 |
-
"""Process multiple images with controlled concurrency."""
|
| 609 |
if not files:
|
| 610 |
return "No images uploaded."
|
| 611 |
-
|
| 612 |
results = []
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
futures = {executor.submit(handle_single_image, f): idx for idx, f in enumerate(files[:8])}
|
| 616 |
-
|
| 617 |
for fut in as_completed(futures):
|
| 618 |
idx = futures[fut]
|
| 619 |
try:
|
| 620 |
-
|
| 621 |
-
results.append(f"**Image {idx+1}:**\n{alt_text}\n")
|
| 622 |
except Exception as e:
|
| 623 |
-
results.append(f"**Image {idx+1}:** Failed
|
| 624 |
-
|
| 625 |
-
# Sort by original order
|
| 626 |
-
results_sorted = sorted(results, key=lambda s: int(re.search(r"Image (\d+):", s).group(1)))
|
| 627 |
-
return "\n".join(results_sorted)
|
| 628 |
|
| 629 |
-
# === Gradio
|
| 630 |
with gr.Blocks(
|
| 631 |
title="π Advanced Product Alt-Text Generator",
|
| 632 |
theme=gr.themes.Soft(),
|
| 633 |
css="""
|
| 634 |
-
.gradio-container { max-width:
|
| 635 |
-
.description-output {
|
| 636 |
-
|
| 637 |
-
line-height: 1.6;
|
| 638 |
-
font-size: 14px;
|
| 639 |
-
}
|
| 640 |
-
.upload-section { border: 2px dashed #ccc; border-radius: 10px; padding: 20px; }
|
| 641 |
"""
|
| 642 |
) as demo:
|
| 643 |
-
|
| 644 |
gr.Markdown("""
|
| 645 |
# πΌοΈ Narrative Alt-Text Generator
|
| 646 |
|
| 647 |
**Generates detailed, flowing descriptions like human-written alt-text:**
|
| 648 |
-
- β
**Detailed narrative style**
|
| 649 |
-
- β
**Character analysis**
|
| 650 |
-
- β
**Scene description**
|
| 651 |
-
- β
**Text integration**
|
| 652 |
-
- β
**Artistic context**
|
| 653 |
-
- β
**Enhanced BLIP
|
| 654 |
-
|
| 655 |
-
Upload images to generate
|
| 656 |
""")
|
| 657 |
-
|
| 658 |
with gr.Row():
|
| 659 |
with gr.Column(scale=1, elem_classes=["upload-section"]):
|
| 660 |
-
file_input = gr.File(
|
| 661 |
-
|
| 662 |
-
|
| 663 |
-
|
| 664 |
-
|
| 665 |
-
)
|
| 666 |
-
generate_btn = gr.Button(
|
| 667 |
-
"π Generate Detailed Alt-Text",
|
| 668 |
-
variant="primary",
|
| 669 |
-
size="lg"
|
| 670 |
-
)
|
| 671 |
-
|
| 672 |
gr.Markdown("""
|
| 673 |
-
**Supported
|
| 674 |
-
**Max
|
| 675 |
-
**
|
| 676 |
""")
|
| 677 |
-
|
| 678 |
with gr.Column(scale=2):
|
| 679 |
-
gallery = gr.Gallery(
|
| 680 |
-
|
| 681 |
-
|
| 682 |
-
|
| 683 |
-
|
| 684 |
-
|
| 685 |
-
|
| 686 |
-
|
| 687 |
-
|
| 688 |
-
label="π Generated Alt-Text Descriptions",
|
| 689 |
-
lines=20,
|
| 690 |
-
show_copy_button=True,
|
| 691 |
-
elem_classes=["description-output"],
|
| 692 |
-
placeholder="Detailed alt-text descriptions will appear here after processing...",
|
| 693 |
-
max_lines=30
|
| 694 |
-
)
|
| 695 |
-
|
| 696 |
-
# Event handlers
|
| 697 |
-
file_input.change(
|
| 698 |
-
fn=preview_images_grid,
|
| 699 |
-
inputs=file_input,
|
| 700 |
-
outputs=gallery,
|
| 701 |
-
show_progress="hidden"
|
| 702 |
-
)
|
| 703 |
-
|
| 704 |
-
generate_btn.click(
|
| 705 |
-
fn=process_images,
|
| 706 |
-
inputs=file_input,
|
| 707 |
-
outputs=output,
|
| 708 |
-
show_progress=True
|
| 709 |
-
)
|
| 710 |
-
|
| 711 |
-
gr.Markdown("""
|
| 712 |
-
---
|
| 713 |
**π§ Technical Features:**
|
| 714 |
-
-
|
| 715 |
-
-
|
| 716 |
-
-
|
| 717 |
-
-
|
| 718 |
-
-
|
| 719 |
-
-
|
| 720 |
-
|
| 721 |
-
**π‘ Best Results:** Use clear, well-lit images with visible branding and text.
|
| 722 |
""")
|
| 723 |
|
| 724 |
if __name__ == "__main__":
|
| 725 |
-
demo.launch(
|
| 726 |
-
server_name="0.0.0.0",
|
| 727 |
-
server_port=7860,
|
| 728 |
-
share=False,
|
| 729 |
-
show_error=True
|
| 730 |
-
)
|
|
|
|
| 54 |
"trucker hat", "baseball cap", "beanie", "cap", "hat", "sunglasses", "watch", "jewelry",
|
| 55 |
"necklace", "bracelet", "ring", "earrings", "bag", "backpack", "purse", "wallet",
|
| 56 |
"belt", "scarf", "gloves", "shoes", "sneakers", "boots", "sandals", "heels",
|
|
|
|
| 57 |
# Electronics & Tech
|
| 58 |
"smartphone", "laptop", "tablet", "headphones", "earbuds", "speaker", "camera",
|
| 59 |
"gaming console", "controller", "keyboard", "mouse", "monitor", "TV", "remote",
|
| 60 |
"charger", "cable", "smartwatch", "fitness tracker", "drone", "robot",
|
|
|
|
| 61 |
# Toys & Games
|
| 62 |
"board game", "card game", "puzzle", "action figure", "doll", "toy car", "LEGO",
|
| 63 |
"building blocks", "stuffed animal", "teddy bear", "toy robot", "game controller",
|
| 64 |
"dice", "playing cards", "collectible card", "miniature figure", "model kit",
|
|
|
|
| 65 |
# Beauty & Personal Care
|
| 66 |
"perfume bottle", "cologne", "lipstick", "mascara", "foundation", "powder compact",
|
| 67 |
"nail polish", "skincare bottle", "lotion", "cream jar", "soap", "shampoo",
|
| 68 |
"conditioner", "toothbrush", "razor", "makeup brush", "mirror", "hair dryer",
|
|
|
|
| 69 |
# Food & Beverages
|
| 70 |
"water bottle", "soda can", "beer bottle", "wine bottle", "coffee mug", "tea cup",
|
| 71 |
"food package", "cereal box", "snack bag", "candy bar", "chocolate", "cookie package",
|
|
|
|
| 72 |
# Books & Media
|
| 73 |
"book", "magazine", "comic book", "DVD", "Blu-ray", "CD", "vinyl record",
|
| 74 |
"notebook", "journal", "planner", "calendar", "poster", "artwork", "photo frame",
|
|
|
|
| 75 |
# Sports & Fitness
|
| 76 |
"basketball", "football", "soccer ball", "tennis ball", "baseball", "golf ball",
|
| 77 |
"dumbbells", "yoga mat", "water bottle", "sports equipment", "helmet", "gloves",
|
|
|
|
| 78 |
# Home & Kitchen
|
| 79 |
"coffee maker", "blender", "toaster", "microwave", "pan", "pot", "plate", "bowl",
|
| 80 |
"cup", "glass", "utensils", "knife", "cutting board", "storage container",
|
|
|
|
| 81 |
# Art & Craft Supplies
|
| 82 |
"paint brush", "marker", "pen", "pencil", "crayon", "paint tube", "canvas",
|
| 83 |
"sketchbook", "ruler", "scissors", "glue", "tape", "stickers",
|
|
|
|
| 84 |
# Packaging & Containers
|
| 85 |
"box", "package", "container", "jar", "bottle", "can", "tube", "pouch", "bag"
|
| 86 |
]
|
|
|
|
| 89 |
# Tech Brands
|
| 90 |
"Apple", "Samsung", "Google", "Microsoft", "Sony", "Nintendo", "PlayStation", "Xbox",
|
| 91 |
"HP", "Dell", "Lenovo", "ASUS", "Acer", "Canon", "Nikon", "GoPro", "DJI",
|
|
|
|
| 92 |
# Fashion & Accessories
|
| 93 |
"Nike", "Adidas", "Puma", "Under Armour", "Reebok", "Converse", "Vans", "New Balance",
|
| 94 |
"Ray-Ban", "Oakley", "Rolex", "Casio", "Fossil", "Michael Kors", "Coach", "Louis Vuitton",
|
|
|
|
| 95 |
# Beauty & Personal Care
|
| 96 |
"L'Oreal", "Maybelline", "MAC", "Sephora", "Clinique", "Estee Lauder", "Chanel",
|
| 97 |
"Dior", "YSL", "Urban Decay", "Too Faced", "NARS", "Fenty Beauty",
|
|
|
|
| 98 |
# Gaming & Toys
|
| 99 |
"Hasbro", "Mattel", "LEGO", "Ravensburger", "Fantasy Flight Games", "Asmodee",
|
| 100 |
"Days of Wonder", "CMON", "Wizards of the Coast", "PokΓ©mon", "Yu-Gi-Oh",
|
|
|
|
| 101 |
# Food & Beverages
|
| 102 |
"Coca-Cola", "Pepsi", "Starbucks", "Red Bull", "Monster", "Gatorade", "Powerade",
|
| 103 |
"Nestle", "Hershey's", "Mars", "Ferrero", "Lindt", "Godiva",
|
|
|
|
| 104 |
# Art & Craft
|
| 105 |
"Crayola", "Sharpie", "Prismacolor", "Winsor & Newton", "Molotow", "Posca",
|
| 106 |
"Staedtler", "Faber-Castell", "Sakura", "Tombow", "Pilot", "Uni-ball",
|
|
|
|
| 107 |
# Specialty/Generic
|
| 108 |
"unknown", "generic", "no brand visible"
|
| 109 |
]
|
|
|
|
| 122 |
]
|
| 123 |
|
| 124 |
# === Enhanced color detection ===
|
|
|
|
| 125 |
CSS3_COLORS = {
|
| 126 |
+
# ... (same mapping as before) ...
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
}
|
| 128 |
|
| 129 |
def get_color_name(rgb_tuple):
|
|
|
|
| 130 |
min_distance = float('inf')
|
| 131 |
closest_color = 'unknown'
|
|
|
|
| 132 |
for color_rgb, color_name in CSS3_COLORS.items():
|
| 133 |
+
dist = sum((a - b)**2 for a, b in zip(rgb_tuple, color_rgb)) ** 0.5
|
| 134 |
+
if dist < min_distance:
|
| 135 |
+
min_distance, closest_color = dist, color_name
|
|
|
|
|
|
|
|
|
|
| 136 |
return closest_color
|
| 137 |
|
| 138 |
def extract_dominant_colors(image: Image.Image, n_colors=3):
|
| 139 |
+
arr = np.array(image.resize((150,150))).reshape(-1,3)
|
| 140 |
+
mask = ~((arr==[255,255,255]).all(axis=1) | (arr==[0,0,0]).all(axis=1))
|
| 141 |
+
arr = arr[mask] if mask.sum()>0 else arr
|
| 142 |
+
if len(arr) < n_colors: return []
|
| 143 |
+
kmeans = KMeans(n_clusters=n_colors, random_state=42, n_init=10).fit(arr)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
colors = []
|
| 145 |
for center in kmeans.cluster_centers_:
|
| 146 |
rgb = tuple(map(int, center))
|
| 147 |
+
colors.append(get_color_name(rgb))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
return colors
|
| 149 |
|
|
|
|
| 150 |
def ask_visual_question(image: Image.Image, question: str):
|
| 151 |
+
if not VQA_AVAILABLE: return ""
|
|
|
|
|
|
|
|
|
|
| 152 |
try:
|
| 153 |
inputs = blip_vqa_processor(image, question, return_tensors="pt").to(device)
|
| 154 |
+
out = blip_vqa_model.generate(**inputs, max_length=50, num_beams=4)
|
| 155 |
+
return blip_vqa_processor.decode(out[0], skip_special_tokens=True).strip()
|
| 156 |
+
except:
|
|
|
|
|
|
|
|
|
|
| 157 |
return ""
|
| 158 |
|
| 159 |
def analyze_with_vqa(image: Image.Image):
|
| 160 |
+
if not VQA_AVAILABLE: return {}
|
| 161 |
+
qs = [
|
|
|
|
|
|
|
|
|
|
| 162 |
"What type of product is this?",
|
| 163 |
"What brand is visible in this image?",
|
| 164 |
"What colors are prominent in this image?",
|
|
|
|
| 170 |
"Are there any logos or symbols visible?",
|
| 171 |
"What is the shape of the main object?"
|
| 172 |
]
|
|
|
|
| 173 |
results = {}
|
| 174 |
+
for q in qs:
|
| 175 |
+
ans = ask_visual_question(image, q)
|
| 176 |
+
if ans and len(ans)>2:
|
| 177 |
+
key = q.lower().rstrip("?")
|
| 178 |
+
results[key] = ans
|
|
|
|
| 179 |
return results
|
| 180 |
|
|
|
|
| 181 |
def extract_comprehensive_ocr(image: Image.Image):
|
|
|
|
| 182 |
try:
|
| 183 |
+
arr = np.array(image)
|
| 184 |
+
gray = cv2.cvtColor(arr, cv2.COLOR_RGB2GRAY)
|
| 185 |
+
pre = []
|
| 186 |
+
pre.append(arr)
|
| 187 |
+
clahe = cv2.createCLAHE(clipLimit=2.0,tileGridSize=(8,8)).apply(gray)
|
| 188 |
+
pre.append(cv2.cvtColor(clahe,cv2.COLOR_GRAY2RGB))
|
| 189 |
+
_,th = cv2.threshold(gray,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)
|
| 190 |
+
pre.append(cv2.cvtColor(th,cv2.COLOR_GRAY2RGB))
|
| 191 |
+
texts=[]
|
| 192 |
+
for p in pre:
|
| 193 |
+
for bbox, txt, conf in ocr_reader.readtext(p,detail=1,paragraph=True):
|
| 194 |
+
if conf>0.3 and len(txt.strip())>1:
|
| 195 |
+
texts.append((txt.strip(),conf))
|
| 196 |
+
seen,set_seen=[],set()
|
| 197 |
+
for t,c in sorted(texts,key=lambda x:x[1],reverse=True):
|
| 198 |
+
cc=re.sub(r'\W+','',t.lower())
|
| 199 |
+
if cc not in set_seen and len(cc)>1:
|
| 200 |
+
seen.append(t); set_seen.add(cc)
|
| 201 |
+
return " | ".join(seen[:5])
|
| 202 |
+
except:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
return ""
|
| 204 |
|
| 205 |
+
# === Zero-shot setup ===
|
| 206 |
def precompute_embeddings():
|
|
|
|
| 207 |
global _product_embeddings, _brand_embeddings, _character_embeddings, _detail_embeddings
|
| 208 |
+
prod = [f"a photo of a {l}" for l in COMPREHENSIVE_PRODUCT_LABELS]
|
| 209 |
+
br = [f"a {b} product" for b in COMPREHENSIVE_BRAND_LABELS]
|
| 210 |
+
char = [f"a photo of a {c}" for c in CHARACTER_LABELS]
|
| 211 |
+
det = [f"an image showing {d}" for d in DETAIL_LABELS]
|
|
|
|
|
|
|
| 212 |
with torch.inference_mode():
|
| 213 |
+
_product_embeddings = clip_model.encode_text(open_clip.tokenize(prod).to(device)); _product_embeddings /= _product_embeddings.norm(dim=-1,keepdim=True)
|
| 214 |
+
_brand_embeddings = clip_model.encode_text(open_clip.tokenize(br).to(device)); _brand_embeddings /= _brand_embeddings.norm(dim=-1,keepdim=True)
|
| 215 |
+
_character_embeddings = clip_model.encode_text(open_clip.tokenize(char).to(device)); _character_embeddings /= _character_embeddings.norm(dim=-1,keepdim=True)
|
| 216 |
+
_detail_embeddings = clip_model.encode_text(open_clip.tokenize(det).to(device)); _detail_embeddings /= _detail_embeddings.norm(dim=-1,keepdim=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 217 |
|
| 218 |
def enhanced_zero_shot_classification(image: Image.Image):
|
|
|
|
| 219 |
with torch.inference_mode():
|
| 220 |
+
img_t = clip_preprocess(image).unsqueeze(0).to(device)
|
| 221 |
+
emb = clip_model.encode_image(img_t)
|
| 222 |
+
emb = emb / emb.norm(dim=-1,keepdim=True)
|
| 223 |
+
res = {}
|
| 224 |
+
for label_list, embeddings, key, topk in [
|
| 225 |
+
(COMPREHENSIVE_PRODUCT_LABELS, _product_embeddings, 'products', 3),
|
| 226 |
+
(COMPREHENSIVE_BRAND_LABELS, _brand_embeddings, 'brands', 3),
|
| 227 |
+
(CHARACTER_LABELS, _character_embeddings, 'characters', 3),
|
| 228 |
+
(DETAIL_LABELS, _detail_embeddings, 'details', 5),
|
| 229 |
+
]:
|
| 230 |
+
logits = 100.0 * (emb @ embeddings.T)
|
| 231 |
+
probs = torch.softmax(logits, dim=-1)
|
| 232 |
+
top = torch.topk(probs, topk, dim=-1)
|
| 233 |
+
res[key] = [(label_list[idx], float(prob)) for prob, idx in zip(top.values[0], top.indices[0])]
|
| 234 |
+
return res
|
| 235 |
+
|
| 236 |
+
# === Updated enhanced captioning ===
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 237 |
def get_enhanced_caption(image: Image.Image):
|
|
|
|
| 238 |
detailed_prompts = [
|
| 239 |
"Provide a detailed, comprehensive description of this image including all visual elements, characters, objects, text, colors, setting, and artistic style.",
|
| 240 |
"Describe this image in great detail, mentioning the title, characters' appearance, clothing, expressions, objects they're holding, background elements, and any text visible.",
|
| 241 |
"Write a thorough description of this cover art or product image, including character details, clothing, facial expressions, objects, background setting, colors, and any visible text or branding.",
|
| 242 |
+
"Analyze this image completely: describe the main subject, their appearance and clothing, any objects or accessories, the background setting, color scheme, artistic style, and all visible text."
|
| 243 |
]
|
| 244 |
+
|
| 245 |
captions = []
|
| 246 |
+
# Baseline unconditional caption
|
|
|
|
| 247 |
try:
|
| 248 |
inputs = blip_processor(image, return_tensors="pt").to(device)
|
| 249 |
with torch.inference_mode():
|
| 250 |
out = blip_model.generate(
|
| 251 |
**inputs,
|
| 252 |
+
max_new_tokens=150,
|
| 253 |
+
num_beams=8,
|
| 254 |
do_sample=True,
|
| 255 |
temperature=0.8,
|
| 256 |
top_p=0.9,
|
|
|
|
| 258 |
no_repeat_ngram_size=2,
|
| 259 |
repetition_penalty=1.1
|
| 260 |
)
|
| 261 |
+
captions.append(blip_processor.decode(out[0], skip_special_tokens=True).strip())
|
|
|
|
| 262 |
except Exception as e:
|
| 263 |
print(f"Unconditional caption error: {e}")
|
| 264 |
+
|
| 265 |
+
# Try all detailed prompts
|
| 266 |
+
for prompt in detailed_prompts:
|
| 267 |
try:
|
| 268 |
inputs = blip_processor(image, text=prompt, return_tensors="pt").to(device)
|
| 269 |
with torch.inference_mode():
|
| 270 |
out = blip_model.generate(
|
| 271 |
**inputs,
|
| 272 |
+
max_new_tokens=200,
|
| 273 |
+
num_beams=8,
|
| 274 |
do_sample=True,
|
| 275 |
temperature=0.7,
|
| 276 |
top_p=0.85,
|
|
|
|
| 278 |
no_repeat_ngram_size=3,
|
| 279 |
repetition_penalty=1.15
|
| 280 |
)
|
| 281 |
+
captions.append(blip_processor.decode(out[0], skip_special_tokens=True).strip())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 282 |
except Exception as e:
|
| 283 |
+
print(f"Detailed caption error for prompt '{prompt[:30]}...': {e}")
|
| 284 |
+
|
|
|
|
|
|
|
| 285 |
if captions:
|
| 286 |
+
best = max(captions, key=lambda s: len(s.split()))
|
| 287 |
+
best = re.sub(r'^(a|an|the)\s+', '', best, flags=re.IGNORECASE).strip().capitalize()
|
| 288 |
+
return best if best.endswith('.') else best + '.'
|
| 289 |
+
return "Unable to generate detailed caption."
|
|
|
|
|
|
|
| 290 |
|
| 291 |
+
# === Comprehensive pipeline ===
|
| 292 |
def comprehensive_image_analysis(image: Image.Image):
|
|
|
|
|
|
|
| 293 |
if max(image.size) > 1024:
|
| 294 |
image = image.copy()
|
| 295 |
+
image.thumbnail((1024,1024), Image.Resampling.LANCZOS)
|
| 296 |
+
|
| 297 |
+
analysis = {
|
| 298 |
+
'caption': get_enhanced_caption(image),
|
| 299 |
+
'vqa': analyze_with_vqa(image) if VQA_AVAILABLE else {},
|
| 300 |
+
'classification': enhanced_zero_shot_classification(image),
|
| 301 |
+
'ocr_text': extract_comprehensive_ocr(image),
|
| 302 |
+
'colors': extract_dominant_colors(image)
|
| 303 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 304 |
return analysis
|
| 305 |
|
|
|
|
| 306 |
def build_narrative_alt_text(analysis):
|
| 307 |
+
main = analysis.get('caption','')
|
| 308 |
+
parts = []
|
| 309 |
+
|
| 310 |
+
ocr = analysis.get('ocr_text','')
|
| 311 |
+
if ocr:
|
| 312 |
+
if any(k in ocr.lower() for k in ['vol','volume']):
|
| 313 |
+
parts.append(f'Text visible includes "{ocr}"')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 314 |
else:
|
| 315 |
+
parts.append(f'Visible text: "{ocr}"')
|
| 316 |
+
|
| 317 |
+
chars = analysis['classification'].get('characters',[])
|
| 318 |
+
if chars and chars[0][1]>0.5 and 'character' in chars[0][0]:
|
| 319 |
+
parts.append(f'The image features {chars[0][0]} art style')
|
| 320 |
+
|
| 321 |
+
cols = analysis.get('colors',[])
|
| 322 |
+
if cols:
|
| 323 |
+
cols_str = ", ".join(cols[:-1]) + (f", and {cols[-1]}" if len(cols)>1 else "")
|
| 324 |
+
parts.append(f'The color palette prominently features {cols_str}')
|
| 325 |
+
|
| 326 |
+
brs = analysis['classification'].get('brands',[])
|
| 327 |
+
vqa_brand = analysis['vqa'].get('what brand is visible in this image','')
|
| 328 |
+
brand = ""
|
| 329 |
+
if brs and brs[0][1]>0.3 and brs[0][0] not in ['unknown','generic','no brand visible']:
|
| 330 |
+
brand = brs[0][0]
|
| 331 |
+
elif vqa_brand and vqa_brand.lower() not in ['no','none','unknown']:
|
| 332 |
+
brand = vqa_brand
|
| 333 |
+
if brand and brand.lower() not in main.lower():
|
| 334 |
+
parts.append(f'Published by {brand}')
|
| 335 |
+
|
| 336 |
+
prods = analysis['classification'].get('products',[])
|
| 337 |
+
if prods and prods[0][1]>0.4 and prods[0][0] in ['comic book','manga','book']:
|
| 338 |
+
main = f'This is a {prods[0][0]} cover. ' + main
|
| 339 |
+
|
| 340 |
+
mat = analysis['vqa'].get('what material does this product appear to be made of','')
|
| 341 |
+
if mat and mat.lower() not in ['paper','unknown']:
|
| 342 |
+
parts.append(f'The material appears to be {mat}')
|
| 343 |
+
|
| 344 |
+
final = main
|
| 345 |
+
for p in parts:
|
| 346 |
+
final += " " + p + "."
|
| 347 |
+
final = re.sub(r'\s+',' ',final).strip()
|
| 348 |
+
if not final.endswith('.'): final += '.'
|
| 349 |
+
return final
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 350 |
|
| 351 |
# === Initialize embeddings ===
|
| 352 |
print("Precomputing embeddings...")
|
| 353 |
precompute_embeddings()
|
| 354 |
print("Embeddings ready!")
|
| 355 |
|
| 356 |
+
# === Image handling ===
|
| 357 |
def handle_single_image(file):
|
|
|
|
| 358 |
try:
|
| 359 |
+
img = Image.open(file.name).convert("RGB")
|
| 360 |
except Exception as e:
|
| 361 |
return f"Failed to open image: {e}"
|
|
|
|
| 362 |
try:
|
| 363 |
+
ana = comprehensive_image_analysis(img)
|
| 364 |
+
return build_narrative_alt_text(ana)
|
|
|
|
| 365 |
except Exception as e:
|
| 366 |
return f"Analysis failed: {e}"
|
| 367 |
|
| 368 |
def preview_images_grid(files):
|
|
|
|
|
|
|
|
|
|
| 369 |
previews = []
|
| 370 |
+
for f in (files or [])[:12]:
|
| 371 |
try:
|
| 372 |
+
img = Image.open(f.name).convert("RGB")
|
| 373 |
+
img.thumbnail((200,200), Image.Resampling.LANCZOS)
|
| 374 |
previews.append(img)
|
| 375 |
except:
|
| 376 |
continue
|
| 377 |
return previews
|
| 378 |
|
| 379 |
def process_images(files):
|
|
|
|
| 380 |
if not files:
|
| 381 |
return "No images uploaded."
|
|
|
|
| 382 |
results = []
|
| 383 |
+
with ThreadPoolExecutor(max_workers=1) as exe:
|
| 384 |
+
futures = {exe.submit(handle_single_image,f): i for i,f in enumerate(files[:8])}
|
|
|
|
|
|
|
| 385 |
for fut in as_completed(futures):
|
| 386 |
idx = futures[fut]
|
| 387 |
try:
|
| 388 |
+
results.append(f"**Image {idx+1}:**\n{fut.result()}\n")
|
|
|
|
| 389 |
except Exception as e:
|
| 390 |
+
results.append(f"**Image {idx+1}:** Failed - {e}\n")
|
| 391 |
+
return "\n".join(sorted(results, key=lambda s:int(re.search(r"Image (\d+):",s).group(1))))
|
|
|
|
|
|
|
|
|
|
| 392 |
|
| 393 |
+
# === Gradio UI ===
|
| 394 |
with gr.Blocks(
|
| 395 |
title="π Advanced Product Alt-Text Generator",
|
| 396 |
theme=gr.themes.Soft(),
|
| 397 |
css="""
|
| 398 |
+
.gradio-container { max-width:1400px!important; }
|
| 399 |
+
.description-output { font-family:'Georgia',serif; line-height:1.6; font-size:14px; }
|
| 400 |
+
.upload-section { border:2px dashed #ccc; border-radius:10px; padding:20px; }
|
|
|
|
|
|
|
|
|
|
|
|
|
| 401 |
"""
|
| 402 |
) as demo:
|
| 403 |
+
|
| 404 |
gr.Markdown("""
|
| 405 |
# πΌοΈ Narrative Alt-Text Generator
|
| 406 |
|
| 407 |
**Generates detailed, flowing descriptions like human-written alt-text:**
|
| 408 |
+
- β
**Detailed narrative style**
|
| 409 |
+
- β
**Character analysis**
|
| 410 |
+
- β
**Scene description**
|
| 411 |
+
- β
**Text integration**
|
| 412 |
+
- β
**Artistic context**
|
| 413 |
+
- β
**Enhanced BLIP prompting**
|
| 414 |
+
|
| 415 |
+
Upload images to generate professional-quality descriptions.
|
| 416 |
""")
|
| 417 |
+
|
| 418 |
with gr.Row():
|
| 419 |
with gr.Column(scale=1, elem_classes=["upload-section"]):
|
| 420 |
+
file_input = gr.File(label="π€ Upload Product Images",
|
| 421 |
+
file_count="multiple",
|
| 422 |
+
file_types=["image"],
|
| 423 |
+
height=200)
|
| 424 |
+
generate_btn = gr.Button("π Generate Detailed Alt-Text", variant="primary", size="lg")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 425 |
gr.Markdown("""
|
| 426 |
+
**Supported:** JPG, PNG, WebP, BMP
|
| 427 |
+
**Max:** 8 images/batch
|
| 428 |
+
**Tip:** Use clear, well-lit photos with visible text/branding.
|
| 429 |
""")
|
| 430 |
+
|
| 431 |
with gr.Column(scale=2):
|
| 432 |
+
gallery = gr.Gallery(label="πΈ Preview", columns=4, rows=2, height=350, object_fit="contain")
|
| 433 |
+
|
| 434 |
+
output = gr.Textbox(label="π Generated Alt-Text", lines=20, show_copy_button=True,
|
| 435 |
+
elem_classes=["description-output"], placeholder="Descriptions appear here...", max_lines=30)
|
| 436 |
+
|
| 437 |
+
file_input.change(fn=preview_images_grid, inputs=file_input, outputs=gallery, show_progress="hidden")
|
| 438 |
+
generate_btn.click(fn=process_images, inputs=file_input, outputs=output, show_progress=True)
|
| 439 |
+
|
| 440 |
+
gr.Markdown("""---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 441 |
**π§ Technical Features:**
|
| 442 |
+
- Enhanced BLIP & detailed prompts
|
| 443 |
+
- Comprehensive CLIP zero-shot
|
| 444 |
+
- Smart OCR preprocessing
|
| 445 |
+
- Color clustering & naming
|
| 446 |
+
- Optional VQA insights
|
| 447 |
+
- Robust error handling
|
|
|
|
|
|
|
| 448 |
""")
|
| 449 |
|
| 450 |
if __name__ == "__main__":
|
| 451 |
+
demo.launch(server_name="0.0.0.0", server_port=7860, share=False, show_error=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|