Workpam commited on
Commit
3971c3f
·
verified ·
1 Parent(s): 1910d62

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +184 -300
app.py CHANGED
@@ -6,16 +6,17 @@ import open_clip
6
  import cv2
7
  import numpy as np
8
  import re
9
- from collections import Counter
10
- from transformers import BlipProcessor, BlipForConditionalGeneration, BlipForQuestionAnswering
11
  from concurrent.futures import ThreadPoolExecutor, as_completed
12
  from sklearn.cluster import KMeans
13
- from difflib import get_close_matches
 
 
 
 
14
 
15
  # === Device setup ===
16
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
17
  torch.backends.cudnn.benchmark = True
18
-
19
  print(f"Using device: {device}")
20
 
21
  # === Model initialization ===
@@ -24,7 +25,7 @@ blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning
24
  blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
25
  blip_model.to(device).eval()
26
 
27
- # BLIP for VQA (Visual Question Answering) - using base model for stability
28
  try:
29
  blip_vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
30
  blip_vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
@@ -45,307 +46,216 @@ clip_model.to(device).eval()
45
  if device.type == "cuda":
46
  clip_model = clip_model.half()
47
 
48
- # OCR reader with multiple languages
49
  ocr_reader = easyocr.Reader(["en", "es", "fr", "de", "it"], gpu=(device.type == "cuda"))
50
 
51
- # === Expanded label sets ===
52
- COMPREHENSIVE_PRODUCT_LABELS = [
53
- # Accessories & Fashion
54
- "trucker hat", "baseball cap", "beanie", "cap", "hat", "sunglasses", "watch", "jewelry",
55
- "necklace", "bracelet", "ring", "earrings", "bag", "backpack", "purse", "wallet",
56
- "belt", "scarf", "gloves", "shoes", "sneakers", "boots", "sandals", "heels",
57
- # Electronics & Tech
58
- "smartphone", "laptop", "tablet", "headphones", "earbuds", "speaker", "camera",
59
- "gaming console", "controller", "keyboard", "mouse", "monitor", "TV", "remote",
60
- "charger", "cable", "smartwatch", "fitness tracker", "drone", "robot",
61
- # Toys & Games
62
- "board game", "card game", "puzzle", "action figure", "doll", "toy car", "LEGO",
63
- "building blocks", "stuffed animal", "teddy bear", "toy robot", "game controller",
64
- "dice", "playing cards", "collectible card", "miniature figure", "model kit",
65
- # Beauty & Personal Care
66
- "perfume bottle", "cologne", "lipstick", "mascara", "foundation", "powder compact",
67
- "nail polish", "skincare bottle", "lotion", "cream jar", "soap", "shampoo",
68
- "conditioner", "toothbrush", "razor", "makeup brush", "mirror", "hair dryer",
69
- # Food & Beverages
70
- "water bottle", "soda can", "beer bottle", "wine bottle", "coffee mug", "tea cup",
71
- "food package", "cereal box", "snack bag", "candy bar", "chocolate", "cookie package",
72
- # Books & Media
73
- "book", "magazine", "comic book", "DVD", "Blu-ray", "CD", "vinyl record",
74
- "notebook", "journal", "planner", "calendar", "poster", "artwork", "photo frame",
75
- # Sports & Fitness
76
- "basketball", "football", "soccer ball", "tennis ball", "baseball", "golf ball",
77
- "dumbbells", "yoga mat", "water bottle", "sports equipment", "helmet", "gloves",
78
- # Home & Kitchen
79
- "coffee maker", "blender", "toaster", "microwave", "pan", "pot", "plate", "bowl",
80
- "cup", "glass", "utensils", "knife", "cutting board", "storage container",
81
- # Art & Craft Supplies
82
- "paint brush", "marker", "pen", "pencil", "crayon", "paint tube", "canvas",
83
- "sketchbook", "ruler", "scissors", "glue", "tape", "stickers",
84
- # Packaging & Containers
85
- "box", "package", "container", "jar", "bottle", "can", "tube", "pouch", "bag"
86
- ]
87
-
88
- COMPREHENSIVE_BRAND_LABELS = [
89
- # Tech Brands
90
- "Apple", "Samsung", "Google", "Microsoft", "Sony", "Nintendo", "PlayStation", "Xbox",
91
- "HP", "Dell", "Lenovo", "ASUS", "Acer", "Canon", "Nikon", "GoPro", "DJI",
92
- # Fashion & Accessories
93
- "Nike", "Adidas", "Puma", "Under Armour", "Reebok", "Converse", "Vans", "New Balance",
94
- "Ray-Ban", "Oakley", "Rolex", "Casio", "Fossil", "Michael Kors", "Coach", "Louis Vuitton",
95
- # Beauty & Personal Care
96
- "L'Oreal", "Maybelline", "MAC", "Sephora", "Clinique", "Estee Lauder", "Chanel",
97
- "Dior", "YSL", "Urban Decay", "Too Faced", "NARS", "Fenty Beauty",
98
- # Gaming & Toys
99
- "Hasbro", "Mattel", "LEGO", "Ravensburger", "Fantasy Flight Games", "Asmodee",
100
- "Days of Wonder", "CMON", "Wizards of the Coast", "Pokémon", "Yu-Gi-Oh",
101
- # Food & Beverages
102
- "Coca-Cola", "Pepsi", "Starbucks", "Red Bull", "Monster", "Gatorade", "Powerade",
103
- "Nestle", "Hershey's", "Mars", "Ferrero", "Lindt", "Godiva",
104
- # Art & Craft
105
- "Crayola", "Sharpie", "Prismacolor", "Winsor & Newton", "Molotow", "Posca",
106
- "Staedtler", "Faber-Castell", "Sakura", "Tombow", "Pilot", "Uni-ball",
107
- # Specialty/Generic
108
- "unknown", "generic", "no brand visible"
109
- ]
110
-
111
- CHARACTER_LABELS = [
112
- "person", "man", "woman", "child", "baby", "elderly person", "athlete", "professional",
113
- "cartoon character", "anime character", "superhero", "mascot", "celebrity", "model",
114
- "fantasy character", "sci-fi character", "historical figure", "fictional character"
115
- ]
116
-
117
- DETAIL_LABELS = [
118
- "text", "logo", "brand name", "price tag", "barcode", "QR code", "warning label",
119
- "ingredients list", "nutritional information", "instructions", "serial number",
120
- "model number", "size information", "color variants", "pattern", "texture",
121
- "metallic finish", "matte finish", "glossy finish", "transparent", "translucent"
122
- ]
123
-
124
- # === Enhanced color detection ===
125
- CSS3_COLORS = {
126
- # ... (same mapping as before) ...
127
  }
128
 
129
- def get_color_name(rgb_tuple):
130
- min_distance = float('inf')
131
- closest_color = 'unknown'
132
- for color_rgb, color_name in CSS3_COLORS.items():
133
- dist = sum((a - b)**2 for a, b in zip(rgb_tuple, color_rgb)) ** 0.5
134
- if dist < min_distance:
135
- min_distance, closest_color = dist, color_name
136
- return closest_color
137
 
138
  def extract_dominant_colors(image: Image.Image, n_colors=3):
139
- arr = np.array(image.resize((150,150))).reshape(-1,3)
140
- mask = ~((arr==[255,255,255]).all(axis=1) | (arr==[0,0,0]).all(axis=1))
141
- arr = arr[mask] if mask.sum()>0 else arr
142
- if len(arr) < n_colors: return []
 
 
 
143
  kmeans = KMeans(n_clusters=n_colors, random_state=42, n_init=10).fit(arr)
144
  colors = []
145
  for center in kmeans.cluster_centers_:
146
  rgb = tuple(map(int, center))
147
- colors.append(get_color_name(rgb))
148
- return colors
149
-
150
- def ask_visual_question(image: Image.Image, question: str):
151
- if not VQA_AVAILABLE: return ""
152
- try:
153
- inputs = blip_vqa_processor(image, question, return_tensors="pt").to(device)
154
- out = blip_vqa_model.generate(**inputs, max_length=50, num_beams=4)
155
- return blip_vqa_processor.decode(out[0], skip_special_tokens=True).strip()
156
- except:
157
- return ""
158
-
159
- def analyze_with_vqa(image: Image.Image):
160
- if not VQA_AVAILABLE: return {}
161
- qs = [
162
- "What type of product is this?",
163
- "What brand is visible in this image?",
164
- "What colors are prominent in this image?",
165
- "Is there any text visible in this image?",
166
- "Are there any people or characters in this image?",
167
- "What is the main object in this image?",
168
- "What material does this product appear to be made of?",
169
- "What is the background of this image?",
170
- "Are there any logos or symbols visible?",
171
- "What is the shape of the main object?"
172
- ]
173
- results = {}
174
- for q in qs:
175
- ans = ask_visual_question(image, q)
176
- if ans and len(ans)>2:
177
- key = q.lower().rstrip("?")
178
- results[key] = ans
179
- return results
180
-
181
- def extract_comprehensive_ocr(image: Image.Image):
182
- try:
183
- arr = np.array(image)
184
- gray = cv2.cvtColor(arr, cv2.COLOR_RGB2GRAY)
185
- pre = []
186
- pre.append(arr)
187
- clahe = cv2.createCLAHE(clipLimit=2.0,tileGridSize=(8,8)).apply(gray)
188
- pre.append(cv2.cvtColor(clahe,cv2.COLOR_GRAY2RGB))
189
- _,th = cv2.threshold(gray,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)
190
- pre.append(cv2.cvtColor(th,cv2.COLOR_GRAY2RGB))
191
- texts=[]
192
- for p in pre:
193
- for bbox, txt, conf in ocr_reader.readtext(p,detail=1,paragraph=True):
194
- if conf>0.3 and len(txt.strip())>1:
195
- texts.append((txt.strip(),conf))
196
- seen,set_seen=[],set()
197
- for t,c in sorted(texts,key=lambda x:x[1],reverse=True):
198
- cc=re.sub(r'\W+','',t.lower())
199
- if cc not in set_seen and len(cc)>1:
200
- seen.append(t); set_seen.add(cc)
201
- return " | ".join(seen[:5])
202
- except:
203
- return ""
204
-
205
- # === Zero-shot setup ===
206
  def precompute_embeddings():
207
- global _product_embeddings, _brand_embeddings, _character_embeddings, _detail_embeddings
208
- prod = [f"a photo of a {l}" for l in COMPREHENSIVE_PRODUCT_LABELS]
209
- br = [f"a {b} product" for b in COMPREHENSIVE_BRAND_LABELS]
210
- char = [f"a photo of a {c}" for c in CHARACTER_LABELS]
211
- det = [f"an image showing {d}" for d in DETAIL_LABELS]
212
  with torch.inference_mode():
213
- _product_embeddings = clip_model.encode_text(open_clip.tokenize(prod).to(device)); _product_embeddings /= _product_embeddings.norm(dim=-1,keepdim=True)
214
- _brand_embeddings = clip_model.encode_text(open_clip.tokenize(br).to(device)); _brand_embeddings /= _brand_embeddings.norm(dim=-1,keepdim=True)
215
- _character_embeddings = clip_model.encode_text(open_clip.tokenize(char).to(device)); _character_embeddings /= _character_embeddings.norm(dim=-1,keepdim=True)
216
- _detail_embeddings = clip_model.encode_text(open_clip.tokenize(det).to(device)); _detail_embeddings /= _detail_embeddings.norm(dim=-1,keepdim=True)
 
 
 
 
217
 
218
  def enhanced_zero_shot_classification(image: Image.Image):
219
  with torch.inference_mode():
220
  img_t = clip_preprocess(image).unsqueeze(0).to(device)
221
- emb = clip_model.encode_image(img_t)
222
- emb = emb / emb.norm(dim=-1,keepdim=True)
223
- res = {}
224
- for label_list, embeddings, key, topk in [
225
- (COMPREHENSIVE_PRODUCT_LABELS, _product_embeddings, 'products', 3),
226
- (COMPREHENSIVE_BRAND_LABELS, _brand_embeddings, 'brands', 3),
227
- (CHARACTER_LABELS, _character_embeddings, 'characters', 3),
228
- (DETAIL_LABELS, _detail_embeddings, 'details', 5),
229
  ]:
230
- logits = 100.0 * (emb @ embeddings.T)
231
  probs = torch.softmax(logits, dim=-1)
232
  top = torch.topk(probs, topk, dim=-1)
233
- res[key] = [(label_list[idx], float(prob)) for prob, idx in zip(top.values[0], top.indices[0])]
234
- return res
 
 
 
 
 
 
 
 
 
 
 
235
 
236
- # === Updated enhanced captioning ===
237
  def get_enhanced_caption(image: Image.Image):
238
- detailed_prompts = [
239
- "Provide a detailed, comprehensive description of this image including all visual elements, characters, objects, text, colors, setting, and artistic style.",
240
- "Describe this image in great detail, mentioning the title, characters' appearance, clothing, expressions, objects they're holding, background elements, and any text visible.",
241
- "Write a thorough description of this cover art or product image, including character details, clothing, facial expressions, objects, background setting, colors, and any visible text or branding.",
242
- "Analyze this image completely: describe the main subject, their appearance and clothing, any objects or accessories, the background setting, color scheme, artistic style, and all visible text."
243
- ]
244
-
245
  captions = []
246
- # Baseline unconditional caption
247
  try:
248
  inputs = blip_processor(image, return_tensors="pt").to(device)
249
- with torch.inference_mode():
250
- out = blip_model.generate(
251
- **inputs,
252
- max_new_tokens=150,
253
- num_beams=8,
254
- do_sample=True,
255
- temperature=0.8,
256
- top_p=0.9,
257
- early_stopping=True,
258
- no_repeat_ngram_size=2,
259
- repetition_penalty=1.1
260
- )
261
  captions.append(blip_processor.decode(out[0], skip_special_tokens=True).strip())
262
  except Exception as e:
263
- print(f"Unconditional caption error: {e}")
264
 
265
- # Try all detailed prompts
266
- for prompt in detailed_prompts:
267
  try:
268
  inputs = blip_processor(image, text=prompt, return_tensors="pt").to(device)
269
- with torch.inference_mode():
270
- out = blip_model.generate(
271
- **inputs,
272
- max_new_tokens=200,
273
- num_beams=8,
274
- do_sample=True,
275
- temperature=0.7,
276
- top_p=0.85,
277
- early_stopping=True,
278
- no_repeat_ngram_size=3,
279
- repetition_penalty=1.15
280
- )
281
  captions.append(blip_processor.decode(out[0], skip_special_tokens=True).strip())
282
  except Exception as e:
283
- print(f"Detailed caption error for prompt '{prompt[:30]}...': {e}")
 
 
 
284
 
285
- if captions:
286
- best = max(captions, key=lambda s: len(s.split()))
287
- best = re.sub(r'^(a|an|the)\s+', '', best, flags=re.IGNORECASE).strip().capitalize()
288
- return best if best.endswith('.') else best + '.'
289
- return "Unable to generate detailed caption."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
290
 
291
  # === Comprehensive pipeline ===
292
  def comprehensive_image_analysis(image: Image.Image):
 
293
  if max(image.size) > 1024:
294
  image = image.copy()
295
  image.thumbnail((1024,1024), Image.Resampling.LANCZOS)
296
 
297
- analysis = {
298
  'caption': get_enhanced_caption(image),
299
- 'vqa': analyze_with_vqa(image) if VQA_AVAILABLE else {},
300
  'classification': enhanced_zero_shot_classification(image),
301
  'ocr_text': extract_comprehensive_ocr(image),
302
  'colors': extract_dominant_colors(image)
303
  }
304
- return analysis
305
 
 
306
  def build_narrative_alt_text(analysis):
307
- main = analysis.get('caption','')
308
  parts = []
309
 
310
- ocr = analysis.get('ocr_text','')
 
311
  if ocr:
312
- if any(k in ocr.lower() for k in ['vol','volume']):
313
- parts.append(f'Text visible includes "{ocr}"')
 
 
 
 
 
314
  else:
315
- parts.append(f'Visible text: "{ocr}"')
 
316
 
317
- chars = analysis['classification'].get('characters',[])
318
- if chars and chars[0][1]>0.5 and 'character' in chars[0][0]:
319
- parts.append(f'The image features {chars[0][0]} art style')
 
320
 
321
- cols = analysis.get('colors',[])
322
- if cols:
323
- cols_str = ", ".join(cols[:-1]) + (f", and {cols[-1]}" if len(cols)>1 else "")
324
- parts.append(f'The color palette prominently features {cols_str}')
325
-
326
- brs = analysis['classification'].get('brands',[])
327
- vqa_brand = analysis['vqa'].get('what brand is visible in this image','')
328
- brand = ""
329
- if brs and brs[0][1]>0.3 and brs[0][0] not in ['unknown','generic','no brand visible']:
330
- brand = brs[0][0]
331
- elif vqa_brand and vqa_brand.lower() not in ['no','none','unknown']:
332
- brand = vqa_brand
333
- if brand and brand.lower() not in main.lower():
334
- parts.append(f'Published by {brand}')
335
-
336
- prods = analysis['classification'].get('products',[])
337
- if prods and prods[0][1]>0.4 and prods[0][0] in ['comic book','manga','book']:
338
- main = f'This is a {prods[0][0]} cover. ' + main
339
-
340
- mat = analysis['vqa'].get('what material does this product appear to be made of','')
341
- if mat and mat.lower() not in ['paper','unknown']:
342
- parts.append(f'The material appears to be {mat}')
343
-
344
- final = main
345
  for p in parts:
346
  final += " " + p + "."
347
- final = re.sub(r'\s+',' ',final).strip()
348
- if not final.endswith('.'): final += '.'
 
349
  return final
350
 
351
  # === Initialize embeddings ===
@@ -353,15 +263,15 @@ print("Precomputing embeddings...")
353
  precompute_embeddings()
354
  print("Embeddings ready!")
355
 
356
- # === Image handling ===
357
  def handle_single_image(file):
358
  try:
359
  img = Image.open(file.name).convert("RGB")
360
  except Exception as e:
361
  return f"Failed to open image: {e}"
362
  try:
363
- ana = comprehensive_image_analysis(img)
364
- return build_narrative_alt_text(ana)
365
  except Exception as e:
366
  return f"Analysis failed: {e}"
367
 
@@ -381,18 +291,17 @@ def process_images(files):
381
  return "No images uploaded."
382
  results = []
383
  with ThreadPoolExecutor(max_workers=1) as exe:
384
- futures = {exe.submit(handle_single_image,f): i for i,f in enumerate(files[:8])}
385
  for fut in as_completed(futures):
386
  idx = futures[fut]
387
  try:
388
  results.append(f"**Image {idx+1}:**\n{fut.result()}\n")
389
  except Exception as e:
390
  results.append(f"**Image {idx+1}:** Failed - {e}\n")
391
- return "\n".join(sorted(results, key=lambda s:int(re.search(r"Image (\d+):",s).group(1))))
392
 
393
- # === Gradio UI ===
394
  with gr.Blocks(
395
- title="🔍 Advanced Product Alt-Text Generator",
396
  theme=gr.themes.Soft(),
397
  css="""
398
  .gradio-container { max-width:1400px!important; }
@@ -400,52 +309,27 @@ with gr.Blocks(
400
  .upload-section { border:2px dashed #ccc; border-radius:10px; padding:20px; }
401
  """
402
  ) as demo:
403
-
404
  gr.Markdown("""
405
- # 🖼️ Narrative Alt-Text Generator
406
-
407
- **Generates detailed, flowing descriptions like human-written alt-text:**
408
- - ✅ **Detailed narrative style**
409
- - ✅ **Character analysis**
410
- - ✅ **Scene description**
411
- - ✅ **Text integration**
412
- - ✅ **Artistic context**
413
- - ✅ **Enhanced BLIP prompting**
414
-
415
- Upload images to generate professional-quality descriptions.
416
  """)
417
 
418
  with gr.Row():
419
  with gr.Column(scale=1, elem_classes=["upload-section"]):
420
- file_input = gr.File(label="📤 Upload Product Images",
421
- file_count="multiple",
422
- file_types=["image"],
423
- height=200)
424
- generate_btn = gr.Button("🚀 Generate Detailed Alt-Text", variant="primary", size="lg")
425
- gr.Markdown("""
426
- **Supported:** JPG, PNG, WebP, BMP
427
- **Max:** 8 images/batch
428
- **Tip:** Use clear, well-lit photos with visible text/branding.
429
- """)
430
-
431
  with gr.Column(scale=2):
432
- gallery = gr.Gallery(label="📸 Preview", columns=4, rows=2, height=350, object_fit="contain")
433
-
434
- output = gr.Textbox(label="📝 Generated Alt-Text", lines=20, show_copy_button=True,
435
- elem_classes=["description-output"], placeholder="Descriptions appear here...", max_lines=30)
436
-
437
- file_input.change(fn=preview_images_grid, inputs=file_input, outputs=gallery, show_progress="hidden")
438
- generate_btn.click(fn=process_images, inputs=file_input, outputs=output, show_progress=True)
439
-
440
- gr.Markdown("""---
441
- **🔧 Technical Features:**
442
- - Enhanced BLIP & detailed prompts
443
- - Comprehensive CLIP zero-shot
444
- - Smart OCR preprocessing
445
- - Color clustering & naming
446
- - Optional VQA insights
447
- - Robust error handling
448
- """)
449
 
450
  if __name__ == "__main__":
451
- demo.launch(server_name="0.0.0.0", server_port=7860, share=False, show_error=True)
 
6
  import cv2
7
  import numpy as np
8
  import re
 
 
9
  from concurrent.futures import ThreadPoolExecutor, as_completed
10
  from sklearn.cluster import KMeans
11
+ from transformers import (
12
+ BlipProcessor,
13
+ BlipForConditionalGeneration,
14
+ BlipForQuestionAnswering
15
+ )
16
 
17
  # === Device setup ===
18
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
19
  torch.backends.cudnn.benchmark = True
 
20
  print(f"Using device: {device}")
21
 
22
  # === Model initialization ===
 
25
  blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
26
  blip_model.to(device).eval()
27
 
28
+ # BLIP for VQA (optional)
29
  try:
30
  blip_vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
31
  blip_vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
 
46
  if device.type == "cuda":
47
  clip_model = clip_model.half()
48
 
49
+ # OCR reader
50
  ocr_reader = easyocr.Reader(["en", "es", "fr", "de", "it"], gpu=(device.type == "cuda"))
51
 
52
+ # === Label sets (unchanged) ===
53
+ COMPREHENSIVE_PRODUCT_LABELS = [ … ] # same as before
54
+ COMPREHENSIVE_BRAND_LABELS = [ … ] # same as before
55
+ CHARACTER_LABELS = [ … ]
56
+ DETAIL_LABELS = [ … ]
57
+
58
+ # === Basic color palette for mapping ===
59
+ BASE_COLORS = {
60
+ 'red': (255, 0, 0),
61
+ 'green': (0, 128, 0),
62
+ 'blue': (0, 0, 255),
63
+ 'yellow': (255, 255, 0),
64
+ 'orange': (255, 165, 0),
65
+ 'purple': (128, 0, 128),
66
+ 'pink': (255, 192, 203),
67
+ 'brown': (165, 42, 42),
68
+ 'gray': (128, 128, 128),
69
+ 'black': (0, 0, 0),
70
+ 'white': (255, 255, 255),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
  }
72
 
73
+ def get_basic_color_name(rgb_tuple):
74
+ """Map any RGB to nearest basic color name."""
75
+ best, best_name = float('inf'), 'unknown'
76
+ for name, ref in BASE_COLORS.items():
77
+ dist = sum((a - b) ** 2 for a, b in zip(rgb_tuple, ref)) ** 0.5
78
+ if dist < best:
79
+ best, best_name = dist, name
80
+ return best_name
81
 
82
  def extract_dominant_colors(image: Image.Image, n_colors=3):
83
+ """Use k-means on downsampled pixels, then map to basic colors."""
84
+ arr = np.array(image.resize((150, 150))).reshape(-1, 3)
85
+ # drop pure black/white
86
+ mask = ~((arr == [255,255,255]).all(axis=1) | (arr == [0,0,0]).all(axis=1))
87
+ arr = arr[mask] if mask.sum() > 0 else arr
88
+ if len(arr) < n_colors:
89
+ return []
90
  kmeans = KMeans(n_clusters=n_colors, random_state=42, n_init=10).fit(arr)
91
  colors = []
92
  for center in kmeans.cluster_centers_:
93
  rgb = tuple(map(int, center))
94
+ colors.append(get_basic_color_name(rgb))
95
+ # dedupe and preserve order
96
+ seen = set()
97
+ result = []
98
+ for c in colors:
99
+ if c not in seen:
100
+ seen.add(c)
101
+ result.append(c)
102
+ return result
103
+
104
+ # === Zero-shot classification setup ===
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
105
  def precompute_embeddings():
106
+ global _prod_emb, _brand_emb, _char_emb, _detail_emb
107
+ prompts_prod = [f"a photo of a {l}" for l in COMPREHENSIVE_PRODUCT_LABELS]
108
+ prompts_brand = [f"a {b} product" for b in COMPREHENSIVE_BRAND_LABELS]
109
+ prompts_char = [f"a photo of a {c}" for c in CHARACTER_LABELS]
110
+ prompts_det = [f"an image showing {d}" for d in DETAIL_LABELS]
111
  with torch.inference_mode():
112
+ _prod_emb = clip_model.encode_text(open_clip.tokenize(prompts_prod).to(device))
113
+ _prod_emb /= _prod_emb.norm(dim=-1, keepdim=True)
114
+ _brand_emb = clip_model.encode_text(open_clip.tokenize(prompts_brand).to(device))
115
+ _brand_emb /= _brand_emb.norm(dim=-1, keepdim=True)
116
+ _char_emb = clip_model.encode_text(open_clip.tokenize(prompts_char).to(device))
117
+ _char_emb /= _char_emb.norm(dim=-1, keepdim=True)
118
+ _det_emb = clip_model.encode_text(open_clip.tokenize(prompts_det).to(device))
119
+ _det_emb /= _det_emb.norm(dim=-1, keepdim=True)
120
 
121
  def enhanced_zero_shot_classification(image: Image.Image):
122
  with torch.inference_mode():
123
  img_t = clip_preprocess(image).unsqueeze(0).to(device)
124
+ img_e = clip_model.encode_image(img_t)
125
+ img_e /= img_e.norm(dim=-1, keepdim=True)
126
+ results = {}
127
+ for labels, emb, key, topk in [
128
+ (COMPREHENSIVE_PRODUCT_LABELS, _prod_emb, 'products', 3),
129
+ (COMPREHENSIVE_BRAND_LABELS, _brand_emb, 'brands', 3),
130
+ (CHARACTER_LABELS, _char_emb, 'characters', 3),
131
+ (DETAIL_LABELS, _det_emb, 'details', 5),
132
  ]:
133
+ logits = (100.0 * img_e @ emb.T)
134
  probs = torch.softmax(logits, dim=-1)
135
  top = torch.topk(probs, topk, dim=-1)
136
+ results[key] = [
137
+ (labels[idx], float(prob))
138
+ for prob, idx in zip(top.values[0], top.indices[0])
139
+ ]
140
+ return results
141
+
142
+ # === BLIP captioning with enhanced prompts ===
143
+ DETAILED_PROMPTS = [
144
+ "Provide a detailed, comprehensive description of this image including all visual elements, characters, objects, text, colors, setting, and artistic style.",
145
+ "Describe this image in great detail, mentioning the title, characters' appearance, clothing, expressions, objects they're holding, background elements, and any text visible.",
146
+ "Write a thorough description of this cover art or product image, including character details, clothing, facial expressions, objects, background setting, colors, and any visible text or branding.",
147
+ "Analyze this image completely: describe the main subject, their appearance and clothing, any objects or accessories, the background setting, color scheme, artistic style, and all visible text."
148
+ ]
149
 
 
150
  def get_enhanced_caption(image: Image.Image):
 
 
 
 
 
 
 
151
  captions = []
152
+ # unconditional baseline
153
  try:
154
  inputs = blip_processor(image, return_tensors="pt").to(device)
155
+ out = blip_model.generate(
156
+ **inputs, max_new_tokens=150, num_beams=8,
157
+ do_sample=True, temperature=0.8, top_p=0.9,
158
+ no_repeat_ngram_size=2, repetition_penalty=1.1
159
+ )
 
 
 
 
 
 
 
160
  captions.append(blip_processor.decode(out[0], skip_special_tokens=True).strip())
161
  except Exception as e:
162
+ print(f"Baseline caption error: {e}")
163
 
164
+ # detailed prompts
165
+ for prompt in DETAILED_PROMPTS:
166
  try:
167
  inputs = blip_processor(image, text=prompt, return_tensors="pt").to(device)
168
+ out = blip_model.generate(
169
+ **inputs, max_new_tokens=200, num_beams=8,
170
+ do_sample=True, temperature=0.7, top_p=0.85,
171
+ no_repeat_ngram_size=3, repetition_penalty=1.15
172
+ )
 
 
 
 
 
 
 
173
  captions.append(blip_processor.decode(out[0], skip_special_tokens=True).strip())
174
  except Exception as e:
175
+ print(f"Prompt caption error: {e}")
176
+
177
+ if not captions:
178
+ return "Unable to generate a caption."
179
 
180
+ # pick the most content-rich (longest) caption
181
+ best = max(captions, key=lambda s: len(s.split()))
182
+ best = re.sub(r'^(a|an|the)\s+', '', best, flags=re.IGNORECASE).strip().capitalize()
183
+ return best if best.endswith('.') else best + '.'
184
+
185
+ # === OCR ===
186
+ def extract_comprehensive_ocr(image: Image.Image):
187
+ try:
188
+ arr = np.array(image)
189
+ gray = cv2.cvtColor(arr, cv2.COLOR_RGB2GRAY)
190
+ preproc = [arr]
191
+ clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8)).apply(gray)
192
+ preproc.append(cv2.cvtColor(clahe, cv2.COLOR_GRAY2RGB))
193
+ _, th = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
194
+ preproc.append(cv2.cvtColor(th, cv2.COLOR_GRAY2RGB))
195
+
196
+ texts = []
197
+ for p in preproc:
198
+ for _, txt, conf in ocr_reader.readtext(p, detail=1, paragraph=True):
199
+ if conf > 0.3 and len(txt.strip()) > 1:
200
+ texts.append((txt.strip(), conf))
201
+ # dedupe
202
+ seen = set()
203
+ unique = []
204
+ for txt, conf in sorted(texts, key=lambda x: x[1], reverse=True):
205
+ key = re.sub(r'\W+', '', txt.lower())
206
+ if key not in seen:
207
+ seen.add(key)
208
+ unique.append(txt)
209
+ return " | ".join(unique[:5])
210
+ except Exception as e:
211
+ print(f"OCR error: {e}")
212
+ return ""
213
 
214
  # === Comprehensive pipeline ===
215
  def comprehensive_image_analysis(image: Image.Image):
216
+ # resize for efficiency
217
  if max(image.size) > 1024:
218
  image = image.copy()
219
  image.thumbnail((1024,1024), Image.Resampling.LANCZOS)
220
 
221
+ return {
222
  'caption': get_enhanced_caption(image),
 
223
  'classification': enhanced_zero_shot_classification(image),
224
  'ocr_text': extract_comprehensive_ocr(image),
225
  'colors': extract_dominant_colors(image)
226
  }
 
227
 
228
+ # === Build narrative alt-text ===
229
  def build_narrative_alt_text(analysis):
230
+ main_desc = analysis['caption']
231
  parts = []
232
 
233
+ # OCR text
234
+ ocr = analysis['ocr_text']
235
  if ocr:
236
+ parts.append(f'Visible text: "{ocr}"')
237
+
238
+ # Dominant colors
239
+ cols = analysis['colors']
240
+ if cols:
241
+ if len(cols) == 1:
242
+ parts.append(f"The primary color is {cols[0]}")
243
  else:
244
+ col_str = ", ".join(cols[:-1]) + f", and {cols[-1]}"
245
+ parts.append(f"The color palette features {col_str}")
246
 
247
+ # Brand (only if high confidence)
248
+ brands = analysis['classification']['brands']
249
+ if brands and brands[0][1] > 0.6:
250
+ parts.append(f"Published by {brands[0][0]}")
251
 
252
+ # Combine
253
+ final = main_desc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
254
  for p in parts:
255
  final += " " + p + "."
256
+ final = re.sub(r'\s+', ' ', final).strip()
257
+ if not final.endswith('.'):
258
+ final += '.'
259
  return final
260
 
261
  # === Initialize embeddings ===
 
263
  precompute_embeddings()
264
  print("Embeddings ready!")
265
 
266
+ # === Gradio interface ===
267
  def handle_single_image(file):
268
  try:
269
  img = Image.open(file.name).convert("RGB")
270
  except Exception as e:
271
  return f"Failed to open image: {e}"
272
  try:
273
+ analysis = comprehensive_image_analysis(img)
274
+ return build_narrative_alt_text(analysis)
275
  except Exception as e:
276
  return f"Analysis failed: {e}"
277
 
 
291
  return "No images uploaded."
292
  results = []
293
  with ThreadPoolExecutor(max_workers=1) as exe:
294
+ futures = {exe.submit(handle_single_image, f): i for i, f in enumerate(files[:8])}
295
  for fut in as_completed(futures):
296
  idx = futures[fut]
297
  try:
298
  results.append(f"**Image {idx+1}:**\n{fut.result()}\n")
299
  except Exception as e:
300
  results.append(f"**Image {idx+1}:** Failed - {e}\n")
301
+ return "\n".join(sorted(results, key=lambda s: int(re.search(r"Image (\d+):", s).group(1))))
302
 
 
303
  with gr.Blocks(
304
+ title="🔍 Optimized Narrative Alt-Text Generator",
305
  theme=gr.themes.Soft(),
306
  css="""
307
  .gradio-container { max-width:1400px!important; }
 
309
  .upload-section { border:2px dashed #ccc; border-radius:10px; padding:20px; }
310
  """
311
  ) as demo:
 
312
  gr.Markdown("""
313
+ # 🖼️ Optimized Alt-Text Generator
314
+
315
+ - Improved basic color mapping
316
+ - Brand only if high confidence (≥60%)
317
+ - Removed unreliable VQA-brand/material
318
+ - K-means → basic colors for clearer palettes
 
 
 
 
 
319
  """)
320
 
321
  with gr.Row():
322
  with gr.Column(scale=1, elem_classes=["upload-section"]):
323
+ inp = gr.File(label="Upload Images", file_count="multiple", file_types=["image"], height=200)
324
+ btn = gr.Button("Generate Alt-Text", variant="primary", size="lg")
 
 
 
 
 
 
 
 
 
325
  with gr.Column(scale=2):
326
+ gal = gr.Gallery(label="Preview", columns=4, rows=2, height=350, object_fit="contain")
327
+
328
+ out = gr.Textbox(label="Generated Alt-Text", lines=20, show_copy_button=True,
329
+ elem_classes=["description-output"], placeholder="...")
330
+
331
+ inp.change(preview_images_grid, inp, gal, show_progress="hidden")
332
+ btn.click(process_images, inp, out, show_progress=True)
 
 
 
 
 
 
 
 
 
 
333
 
334
  if __name__ == "__main__":
335
+ demo.launch(server_name="0.0.0.0", server_port=7860, share=False)