Workpam commited on
Commit
d78450a
Β·
verified Β·
1 Parent(s): bca956f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +530 -331
app.py CHANGED
@@ -7,416 +7,615 @@ import cv2
7
  import numpy as np
8
  import re
9
  from collections import Counter
10
- from transformers import BlipProcessor, BlipForConditionalGeneration
11
  from concurrent.futures import ThreadPoolExecutor, as_completed
 
 
 
12
 
13
  # === Device setup ===
14
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
15
- torch.backends.cudnn.benchmark = True # speeds up variable-size inputs
 
 
16
 
17
- # === Model initialization (once) ===
18
  # BLIP for captions
19
- blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
20
- blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
21
  blip_model.to(device).eval()
 
 
 
 
 
 
22
  if device.type == "cuda":
23
  blip_model = blip_model.half()
 
24
 
25
  # CLIP for zero-shot classification
26
- clip_model, _, clip_preprocess = open_clip.create_model_and_transforms("ViT-B-32", pretrained="openai")
27
  clip_model.to(device).eval()
28
  if device.type == "cuda":
29
  clip_model = clip_model.half()
30
 
31
- # OCR reader (English only, can add other langs)
32
- ocr_reader = easyocr.Reader(["en"], gpu=(device.type == "cuda"))
33
-
34
- # === Default label sets (extendable) ===
35
-
36
- from difflib import get_close_matches
37
-
38
- # a small reservoir of likely words to help clean OCR; expand as needed
39
- OCR_DICTIONARY = [
40
- "the", "world", "of", "blame", "man", "watch", "pink", "Mattel", "logo", "edition",
41
- "limited", "series", "heroism", "teamwork", "flash", "point"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  ]
43
 
44
- def normalize_ocr_text(raw: str):
45
- tokens = re.findall(r"\w+", raw.lower())
46
- cleaned_tokens = []
47
- for t in tokens:
48
- match = get_close_matches(t, OCR_DICTIONARY, n=1, cutoff=0.8)
49
- cleaned_tokens.append(match[0] if match else t)
50
- return " ".join(cleaned_tokens).capitalize()
51
-
52
- DEFAULT_PRODUCT_LABELS = [
53
- "trucker hat", "baseball cap", "cap", "board game box", "product box", "doll",
54
- "brush marker", "paint marker", "marker", "card game box", "bottle", "package",
55
- "headphones", "smartphone", "laptop", "watch", "shoe", "sneakers", "book", "skincare bottle",
56
- "cosmetic jar", "perfume bottle", "toy figure", "sporting equipment", "water bottle"
57
- ]
58
- DEFAULT_BRAND_LABELS = [
59
- "Molotow", "Flash Point", "Evergreen", "Hasbro", "Mattel", "Ravensburger",
60
- "Fantasy Flight", "Asmodee", "Days of Wonder", "CMON", "Nike", "Adidas", "Sony", "Samsung"
 
 
 
 
 
 
 
 
 
 
61
  ]
62
 
63
- # Precompute text embeddings for zero-shot (with simple prompt templates)
64
- def embed_texts(texts):
65
- with torch.inference_mode():
66
- tokens = open_clip.tokenize(texts).to(device)
67
- emb = clip_model.encode_text(tokens)
68
- emb = emb / emb.norm(dim=-1, keepdim=True)
69
- return emb # shape (len(texts), dim)
70
-
71
- _product_prompts = [f"a photo of a {p}" for p in DEFAULT_PRODUCT_LABELS]
72
- _brand_prompts = [f"a product from {b}" for b in DEFAULT_BRAND_LABELS] + ["unknown"]
73
-
74
- _product_label_emb = embed_texts(_product_prompts)
75
- _brand_label_emb = embed_texts(_brand_prompts)
76
- _product_labels_extended = DEFAULT_PRODUCT_LABELS.copy()
77
- _brand_labels_extended = DEFAULT_BRAND_LABELS + ["unknown"]
78
-
79
- # === Named colors for approximation (vectorized) ===
80
- NAMED_COLORS = {
81
- "white": (255, 255, 255),
82
- "black": (0, 0, 0),
83
- "beige": (245, 245, 220),
84
- "orange": (255, 165, 0),
85
- "red": (220, 20, 60),
86
- "blue": (30, 144, 255),
87
- "green": (34, 139, 34),
88
- "yellow": (255, 215, 0),
89
- "brown": (139, 69, 19),
90
- "gray": (128, 128, 128),
91
- "pink": (255, 192, 203),
92
- "purple": (128, 0, 128),
93
- }
94
- _color_names = np.array(list(NAMED_COLORS.keys()))
95
- _color_values = np.array(list(NAMED_COLORS.values())) # (N,3)
96
-
97
- def nearest_color_name(rgb):
98
- rgb_arr = np.array(rgb, dtype=int)
99
- dists = np.sum((_color_values - rgb_arr) ** 2, axis=1)
100
- idx = int(np.argmin(dists))
101
- return _color_names[idx]
102
-
103
- # === Utility / gating helpers ===
104
-
105
- def is_solid_background(image: Image.Image, color_threshold=0.85, edge_threshold=0.03):
106
- small = cv2.resize(np.array(image), (100, 100), interpolation=cv2.INTER_AREA)
107
- pixels = small.reshape(-1, 3)
108
- tup = [tuple(p) for p in pixels]
109
- cnt = Counter(tup)
110
- dominant_frac = cnt.most_common(1)[0][1] / len(tup)
111
- gray = cv2.cvtColor(small, cv2.COLOR_RGB2GRAY)
112
- edges = cv2.Canny(gray, 100, 200)
113
- edge_density = np.mean(edges > 0)
114
- return dominant_frac >= color_threshold and edge_density <= edge_threshold
115
 
116
- def should_run_ocr(image: Image.Image, edge_density_threshold=0.01):
117
- gray = cv2.cvtColor(np.array(image.resize((200, 200))), cv2.COLOR_RGB2GRAY)
118
- edges = cv2.Canny(gray, 100, 200)
119
- edge_density = np.mean(edges > 0)
120
- return edge_density >= edge_density_threshold # low edge density likely no text/logo
 
121
 
122
- def extract_and_clean_ocr(image: Image.Image):
123
- if not should_run_ocr(image):
124
- return ""
125
- arr = np.array(image)
126
  try:
127
- data = ocr_reader.readtext(arr, detail=1)
128
- except Exception:
129
- return ""
130
- texts = []
131
- for bbox, txt, conf in data:
132
- if conf >= 0.5 and txt.strip():
133
- texts.append(txt.strip())
134
- if not texts:
135
- return ""
136
- raw = " ".join(texts).lower()
137
-
138
- # Flash Point tagline normalization
139
- if "thrilling" in raw and "heroism" in raw and "teamwork" in raw:
140
- return "A thrilling game of heroism and teamwork, where every choice matters."
141
-
142
- # Common OCR artifact fixes
143
- replacements = {
144
- "where ever": "where every",
145
- "teamw ork": "teamwork",
146
- "choic e": "choice",
147
- "choic es": "choices",
148
- "her oism": "heroism",
149
- "t hrilling": "thrilling",
150
- "heroi sm": "heroism",
151
- "team work": "teamwork"
152
- }
153
- for k, v in replacements.items():
154
- raw = raw.replace(k, v)
155
-
156
- return raw.strip().capitalize()
157
-
158
- # === Captioning with BLIP (ensemble of prompts) ===
 
 
 
 
 
 
 
159
 
160
- def get_blip_caption(image: Image.Image, prompt: str = None):
161
- def generate_with(p):
162
- inputs = blip_processor(image, text=p, return_tensors="pt").to(device)
 
 
163
  with torch.inference_mode():
164
- out = blip_model.generate(
165
- **inputs,
166
- max_new_tokens=60 if prompt is None else 40,
167
- num_beams=4,
168
- do_sample=False,
169
- early_stopping=True,
170
- no_repeat_ngram_size=2
171
- )
172
- caption = blip_processor.decode(out[0], skip_special_tokens=True).strip()
173
- return caption
174
-
175
- base_prompt = prompt or "Describe the product in the image including its type, visible text/logo, brand if any, and key visual features."
176
- caption = generate_with(base_prompt)
177
-
178
- # If caption is too short or just repeats the instruction, retry with a different phrasing
179
- if (
180
- len(caption.split()) < 3
181
- or base_prompt.lower() in caption.lower()
182
- or re.search(r"describe the product", caption, flags=re.IGNORECASE)
183
- ):
184
- alt_prompt = "Provide a concise descriptive caption of the product, its brand, and distinguishing visual attributes."
185
- caption = generate_with(alt_prompt)
186
-
187
- caption = re.sub(r'^(the|a|an)\s+', '', caption, flags=re.IGNORECASE).strip()
188
- return caption.capitalize()
189
-
190
-
191
- def get_background_caption(image: Image.Image):
192
- prompt = "Describe only the background of this image, excluding the main object."
193
- caption = get_blip_caption(image, prompt=prompt)
194
- if "describe only the background" in caption.lower():
195
  return ""
196
- return caption
197
 
198
- # === CLIP zero-shot inference using cached embeddings ===
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
199
 
200
- def zero_shot_clip(image: Image.Image, label_embs, labels):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
201
  with torch.inference_mode():
202
  img_tensor = clip_preprocess(image).unsqueeze(0).to(device)
203
  img_emb = clip_model.encode_image(img_tensor)
204
  img_emb = img_emb / img_emb.norm(dim=-1, keepdim=True)
205
- logits = 100.0 * img_emb @ label_embs.T # (1, num_labels)
206
- probs = torch.softmax(logits, dim=-1)
207
- best_idx = int(probs.argmax())
208
- return labels[best_idx], float(probs[0, best_idx])
209
-
210
- # === Color extraction ===
211
-
212
- def dominant_color_of_image(image: Image.Image):
213
- arr = np.array(image).reshape(-1, 3)
214
- avg = tuple(arr.mean(axis=0).astype(int).tolist())
215
- return avg
216
-
217
- def dominant_color_for_object(image: Image.Image, product_type: str):
218
- if product_type and "box" in product_type.lower():
219
- return None
220
- rgb = dominant_color_of_image(image)
221
- name = nearest_color_name(rgb)
222
- if name == "white":
223
- return None
224
- return name
225
-
226
- # === Structured inference combining signals ===
227
-
228
- def infer_product_info(image: Image.Image):
229
- # Resize to reasonable size early for speed
230
- if max(image.size) > 800:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
231
  image = image.copy()
232
- image.thumbnail((800, 800), Image.Resampling.LANCZOS)
233
-
234
- # Captions (ensemble: base and alternate phrasing)
235
- caption1 = get_blip_caption(image)
236
- caption2 = get_blip_caption(image, prompt="Provide a detailed description of the product, including its type, brand cues, visible text/logo, and distinguishing features.")
237
- # Choose longer or merge heuristically
238
- caption = caption2 if len(caption2) > len(caption1) else caption1
239
-
240
- # Zero-shot product type and brand
241
- product_type, prod_conf = zero_shot_clip(image, _product_label_emb, _product_labels_extended)
242
- brand, brand_conf = zero_shot_clip(image, _brand_label_emb, _brand_labels_extended)
243
-
244
- # Dominant color (skip boxes)
245
- color = dominant_color_for_object(image, product_type or "")
246
-
247
- # Background description if not solid
248
- background_desc = ""
249
- if not is_solid_background(image):
250
- background_desc = get_background_caption(image)
251
-
252
- return {
253
- "brand": brand if brand != "unknown" else None,
254
- "brand_confidence": brand_conf,
255
- "product_type": product_type,
256
- "product_confidence": prod_conf,
257
- "caption": caption,
258
- "dominant_color": color,
259
- "background": background_desc,
260
- }
261
-
262
- # === Alt-text builder ===
263
-
264
- def build_alt_text(structured, ocr_text=""):
265
  parts = []
266
- # Brand and product
267
- if structured["brand"]:
268
- parts.append(f"{structured['brand']}")
269
- if structured["product_type"]:
270
- parts.append(f"{structured['product_type']}")
271
- # Color
272
- if structured["dominant_color"]:
273
- parts.append(f"in {structured['dominant_color']}")
274
- # OCR
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
275
  if ocr_text:
276
- parts.append(f'Text visible: "{ocr_text}"')
277
- # Caption
278
- if structured["caption"]:
279
- parts.append(f"Caption: {structured['caption']}")
280
- # Background
281
- if structured["background"]:
282
- parts.append(f"Background: {structured['background']}")
283
-
284
- alt = "; ".join(parts).strip()
285
- if not alt.endswith("."):
286
- alt += "."
287
- return alt
288
-
289
- # === Flash Point special override ===
290
-
291
- def flash_point_override(image: Image.Image, ocr_text: str, fallback_caption: str):
292
- combined_lower = (fallback_caption + " " + (ocr_text or "")).lower()
293
- if "flash point" in combined_lower or "legacy of flame" in combined_lower:
294
- parts = ['Flash Point: Legacy of Flame board game box cover', 'featuring a firefighter amid flames']
295
- if "thrilling" in (ocr_text or "").lower() and "heroism" in (ocr_text or "").lower():
296
- parts.append('tagline β€œA thrilling game of heroism and teamwork, where every choice matters.”')
297
- elif ocr_text:
298
- parts.append(f'tagline "{ocr_text}"')
299
- if not is_solid_background(image):
300
- bg = get_background_caption(image)
301
- if bg:
302
- parts.append(f"Background: {bg}")
303
- alt = "; ".join(parts)
304
- if not alt.endswith("."):
305
- alt += "."
306
- return alt
307
- return None
308
-
309
- # === Image processing pipeline (single image) ===
310
-
311
  def handle_single_image(file):
 
312
  try:
313
  image = Image.open(file.name).convert("RGB")
314
  except Exception as e:
315
  return f"Failed to open image: {e}"
316
-
317
- if max(image.size) > 800:
318
- image.thumbnail((800, 800), Image.Resampling.LANCZOS)
319
-
320
- # OCR
321
- ocr_text = extract_and_clean_ocr(image)
322
-
323
- # BLIP fallback caption (for override)
324
- fallback_caption = get_blip_caption(image)
325
-
326
- # Flash Point override check
327
- fp = flash_point_override(image, ocr_text, fallback_caption)
328
- if fp:
329
- return fp
330
-
331
- # Structured inference
332
- structured = infer_product_info(image)
333
-
334
- # Build alt-text
335
- alt = build_alt_text(structured, ocr_text=ocr_text)
336
-
337
- return alt
338
-
339
- # === Gradio UI helpers ===
340
 
341
  def preview_images_grid(files):
 
342
  if not files:
343
  return []
344
  previews = []
345
- for file in files[:15]:
346
  try:
347
  img = Image.open(file.name).convert("RGB")
348
- img.thumbnail((250, 250), Image.Resampling.LANCZOS)
349
  previews.append(img)
350
  except:
351
  continue
352
  return previews
353
 
354
- def process_images(files, style):
 
355
  if not files:
356
  return "No images uploaded."
 
357
  results = []
358
- # Controlled concurrency: 2 workers to avoid GPU contention
359
- with ThreadPoolExecutor(max_workers=2) as executor:
360
- futures = {executor.submit(handle_single_image, f): idx for idx, f in enumerate(files[:10])}
 
361
  for fut in as_completed(futures):
362
  idx = futures[fut]
363
  try:
364
- alt = fut.result()
365
- results.append(f"Image {idx+1}: {alt}")
366
  except Exception as e:
367
- results.append(f"Image {idx+1}: Failed to describe ({e})")
368
- # Sort by original index to preserve order
369
- results_sorted = sorted(results, key=lambda s: int(re.match(r"Image (\d+):", s).group(1)))
370
- return "\n\n".join(results_sorted)
 
371
 
 
372
  with gr.Blocks(
373
- title="πŸ›οΈ Generalized Product Alt-text Generator",
374
  theme=gr.themes.Soft(),
375
  css="""
376
- .gradio-container { max-width: 1200px !important; }
377
- .description-output { font-family: 'Georgia', serif; line-height:1.5; }
 
 
 
 
 
378
  """
379
  ) as demo:
380
- gr.Markdown(
381
- """
382
- ### Generalized Product Alt-text Generator
383
- Uploads any product image and returns structured, objective, descriptive alt text including brand, product type, visible text/logo, key features, colors, context, and background when relevant.
384
- """
385
- )
 
 
 
 
 
 
 
 
 
 
386
  with gr.Row():
387
- with gr.Column(scale=1):
388
  file_input = gr.File(
389
- label="Upload product images",
390
  file_count="multiple",
391
  file_types=["image"],
392
- height=160,
393
  )
394
- generate_btn = gr.Button("Generate Alt Text", variant="primary")
 
 
 
 
 
 
 
 
 
 
 
395
  with gr.Column(scale=2):
396
  gallery = gr.Gallery(
397
- label="Image Preview",
398
  columns=4,
399
  rows=2,
400
- height=300,
401
  object_fit="contain"
402
  )
 
403
  output = gr.Textbox(
404
- label="Generated Alt Text",
405
- lines=16,
406
  show_copy_button=True,
407
  elem_classes=["description-output"],
408
- placeholder="Structured alt text will appear here..."
 
 
 
 
 
 
 
 
 
409
  )
410
- file_input.change(fn=preview_images_grid, inputs=file_input, outputs=gallery, show_progress=True)
411
- generate_btn.click(fn=process_images, inputs=[file_input, gr.State("auto")], outputs=output, show_progress=True)
412
- gr.Markdown(
413
- """
414
- **Notes:**
415
- - Special override ensures Flash Point cover gets accurate templated caption.
416
- - Solid white backgrounds are ignored unless scenic; scenic backgrounds get a separate description.
417
- - Extend `DEFAULT_BRAND_LABELS` and `DEFAULT_PRODUCT_LABELS` for better domain coverage.
418
- """
419
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
420
 
421
  if __name__ == "__main__":
422
- demo.launch(server_name="0.0.0.0", server_port=7860)
 
 
 
 
 
 
7
  import numpy as np
8
  import re
9
  from collections import Counter
10
+ from transformers import BlipProcessor, BlipForConditionalGeneration, BlipProcessor, BlipForQuestionAnswering
11
  from concurrent.futures import ThreadPoolExecutor, as_completed
12
+ from sklearn.cluster import KMeans
13
+ import webcolors
14
+ from difflib import get_close_matches
15
 
16
  # === Device setup ===
17
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
18
+ torch.backends.cudnn.benchmark = True
19
+
20
+ print(f"Using device: {device}")
21
 
22
+ # === Model initialization ===
23
  # BLIP for captions
24
+ blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large")
25
+ blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
26
  blip_model.to(device).eval()
27
+
28
+ # BLIP for VQA (Visual Question Answering)
29
+ blip_vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
30
+ blip_vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
31
+ blip_vqa_model.to(device).eval()
32
+
33
  if device.type == "cuda":
34
  blip_model = blip_model.half()
35
+ blip_vqa_model = blip_vqa_model.half()
36
 
37
  # CLIP for zero-shot classification
38
+ clip_model, _, clip_preprocess = open_clip.create_model_and_transforms("ViT-L-14", pretrained="openai")
39
  clip_model.to(device).eval()
40
  if device.type == "cuda":
41
  clip_model = clip_model.half()
42
 
43
+ # OCR reader with multiple languages
44
+ ocr_reader = easyocr.Reader(["en", "es", "fr", "de", "it"], gpu=(device.type == "cuda"))
45
+
46
+ # === Expanded label sets ===
47
+ COMPREHENSIVE_PRODUCT_LABELS = [
48
+ # Accessories & Fashion
49
+ "trucker hat", "baseball cap", "beanie", "cap", "hat", "sunglasses", "watch", "jewelry",
50
+ "necklace", "bracelet", "ring", "earrings", "bag", "backpack", "purse", "wallet",
51
+ "belt", "scarf", "gloves", "shoes", "sneakers", "boots", "sandals", "heels",
52
+
53
+ # Electronics & Tech
54
+ "smartphone", "laptop", "tablet", "headphones", "earbuds", "speaker", "camera",
55
+ "gaming console", "controller", "keyboard", "mouse", "monitor", "TV", "remote",
56
+ "charger", "cable", "smartwatch", "fitness tracker", "drone", "robot",
57
+
58
+ # Toys & Games
59
+ "board game", "card game", "puzzle", "action figure", "doll", "toy car", "LEGO",
60
+ "building blocks", "stuffed animal", "teddy bear", "toy robot", "game controller",
61
+ "dice", "playing cards", "collectible card", "miniature figure", "model kit",
62
+
63
+ # Beauty & Personal Care
64
+ "perfume bottle", "cologne", "lipstick", "mascara", "foundation", "powder compact",
65
+ "nail polish", "skincare bottle", "lotion", "cream jar", "soap", "shampoo",
66
+ "conditioner", "toothbrush", "razor", "makeup brush", "mirror", "hair dryer",
67
+
68
+ # Food & Beverages
69
+ "water bottle", "soda can", "beer bottle", "wine bottle", "coffee mug", "tea cup",
70
+ "food package", "cereal box", "snack bag", "candy bar", "chocolate", "cookie package",
71
+
72
+ # Books & Media
73
+ "book", "magazine", "comic book", "DVD", "Blu-ray", "CD", "vinyl record",
74
+ "notebook", "journal", "planner", "calendar", "poster", "artwork", "photo frame",
75
+
76
+ # Sports & Fitness
77
+ "basketball", "football", "soccer ball", "tennis ball", "baseball", "golf ball",
78
+ "dumbbells", "yoga mat", "water bottle", "sports equipment", "helmet", "gloves",
79
+
80
+ # Home & Kitchen
81
+ "coffee maker", "blender", "toaster", "microwave", "pan", "pot", "plate", "bowl",
82
+ "cup", "glass", "utensils", "knife", "cutting board", "storage container",
83
+
84
+ # Art & Craft Supplies
85
+ "paint brush", "marker", "pen", "pencil", "crayon", "paint tube", "canvas",
86
+ "sketchbook", "ruler", "scissors", "glue", "tape", "stickers",
87
+
88
+ # Packaging & Containers
89
+ "box", "package", "container", "jar", "bottle", "can", "tube", "pouch", "bag"
90
  ]
91
 
92
+ COMPREHENSIVE_BRAND_LABELS = [
93
+ # Tech Brands
94
+ "Apple", "Samsung", "Google", "Microsoft", "Sony", "Nintendo", "PlayStation", "Xbox",
95
+ "HP", "Dell", "Lenovo", "ASUS", "Acer", "Canon", "Nikon", "GoPro", "DJI",
96
+
97
+ # Fashion & Accessories
98
+ "Nike", "Adidas", "Puma", "Under Armour", "Reebok", "Converse", "Vans", "New Balance",
99
+ "Ray-Ban", "Oakley", "Rolex", "Casio", "Fossil", "Michael Kors", "Coach", "Louis Vuitton",
100
+
101
+ # Beauty & Personal Care
102
+ "L'Oreal", "Maybelline", "MAC", "Sephora", "Clinique", "Estee Lauder", "Chanel",
103
+ "Dior", "YSL", "Urban Decay", "Too Faced", "NARS", "Fenty Beauty",
104
+
105
+ # Gaming & Toys
106
+ "Hasbro", "Mattel", "LEGO", "Ravensburger", "Fantasy Flight Games", "Asmodee",
107
+ "Days of Wonder", "CMON", "Wizards of the Coast", "PokΓ©mon", "Yu-Gi-Oh",
108
+
109
+ # Food & Beverages
110
+ "Coca-Cola", "Pepsi", "Starbucks", "Red Bull", "Monster", "Gatorade", "Powerade",
111
+ "Nestle", "Hershey's", "Mars", "Ferrero", "Lindt", "Godiva",
112
+
113
+ # Art & Craft
114
+ "Crayola", "Sharpie", "Prismacolor", "Winsor & Newton", "Molotow", "Posca",
115
+ "Staedtler", "Faber-Castell", "Sakura", "Tombow", "Pilot", "Uni-ball",
116
+
117
+ # Specialty/Generic
118
+ "unknown", "generic", "no brand visible"
119
  ]
120
 
121
+ CHARACTER_LABELS = [
122
+ "person", "man", "woman", "child", "baby", "elderly person", "athlete", "professional",
123
+ "cartoon character", "anime character", "superhero", "mascot", "celebrity", "model",
124
+ "fantasy character", "sci-fi character", "historical figure", "fictional character"
125
+ ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
126
 
127
+ DETAIL_LABELS = [
128
+ "text", "logo", "brand name", "price tag", "barcode", "QR code", "warning label",
129
+ "ingredients list", "nutritional information", "instructions", "serial number",
130
+ "model number", "size information", "color variants", "pattern", "texture",
131
+ "metallic finish", "matte finish", "glossy finish", "transparent", "translucent"
132
+ ]
133
 
134
+ # === Enhanced color detection ===
135
+ def get_color_name(rgb_tuple):
 
 
136
  try:
137
+ closest_name = webcolors.rgb_to_name(rgb_tuple)
138
+ except ValueError:
139
+ # Find closest color
140
+ min_colours = {}
141
+ for key, name in webcolors.CSS3_HEX_TO_NAMES.items():
142
+ r_c, g_c, b_c = webcolors.hex_to_rgb(key)
143
+ rd = (r_c - rgb_tuple[0]) ** 2
144
+ gd = (g_c - rgb_tuple[1]) ** 2
145
+ bd = (b_c - rgb_tuple[2]) ** 2
146
+ min_colours[(rd + gd + bd)] = name
147
+ closest_name = min_colours[min(min_colours.keys())]
148
+ return closest_name
149
+
150
+ def extract_dominant_colors(image: Image.Image, n_colors=3):
151
+ """Extract dominant colors using K-means clustering."""
152
+ img_array = np.array(image.resize((150, 150)))
153
+ img_array = img_array.reshape(-1, 3)
154
+
155
+ # Remove pure white and black pixels for better color analysis
156
+ mask = ~((img_array == [255, 255, 255]).all(axis=1) | (img_array == [0, 0, 0]).all(axis=1))
157
+ if mask.sum() > 0:
158
+ img_array = img_array[mask]
159
+
160
+ if len(img_array) < n_colors:
161
+ return []
162
+
163
+ kmeans = KMeans(n_clusters=n_colors, random_state=42, n_init=10)
164
+ kmeans.fit(img_array)
165
+
166
+ colors = []
167
+ for center in kmeans.cluster_centers_:
168
+ rgb = tuple(map(int, center))
169
+ try:
170
+ color_name = get_color_name(rgb)
171
+ colors.append(color_name)
172
+ except:
173
+ colors.append(f"RGB{rgb}")
174
+
175
+ return colors
176
 
177
+ # === Enhanced VQA for detailed analysis ===
178
+ def ask_visual_question(image: Image.Image, question: str):
179
+ """Use BLIP VQA to ask specific questions about the image."""
180
+ try:
181
+ inputs = blip_vqa_processor(image, question, return_tensors="pt").to(device)
182
  with torch.inference_mode():
183
+ out = blip_vqa_model.generate(**inputs, max_length=50, num_beams=4, early_stopping=True)
184
+ answer = blip_vqa_processor.decode(out[0], skip_special_tokens=True)
185
+ return answer.strip()
186
+ except Exception as e:
187
+ print(f"VQA error: {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
188
  return ""
 
189
 
190
+ def analyze_with_vqa(image: Image.Image):
191
+ """Analyze image using multiple VQA questions."""
192
+ questions = [
193
+ "What type of product is this?",
194
+ "What brand is visible in this image?",
195
+ "What colors are prominent in this image?",
196
+ "Is there any text visible in this image?",
197
+ "Are there any people or characters in this image?",
198
+ "What is the main object in this image?",
199
+ "What material does this product appear to be made of?",
200
+ "What is the background of this image?",
201
+ "Are there any logos or symbols visible?",
202
+ "What is the shape of the main object?"
203
+ ]
204
+
205
+ results = {}
206
+ for question in questions:
207
+ answer = ask_visual_question(image, question)
208
+ if answer and len(answer) > 2: # Filter out very short/meaningless answers
209
+ key = question.lower().replace("what ", "").replace("is ", "").replace("are ", "").replace("?", "")
210
+ results[key] = answer
211
+
212
+ return results
213
+
214
+ # === Enhanced OCR with better text processing ===
215
+ def extract_comprehensive_ocr(image: Image.Image):
216
+ """Enhanced OCR with better preprocessing and post-processing."""
217
+ try:
218
+ # Convert to numpy array
219
+ img_array = np.array(image)
220
+
221
+ # Preprocessing for better OCR
222
+ gray = cv2.cvtColor(img_array, cv2.COLOR_RGB2GRAY)
223
+
224
+ # Multiple preprocessing approaches
225
+ preprocessed_images = []
226
+
227
+ # Original
228
+ preprocessed_images.append(img_array)
229
+
230
+ # High contrast
231
+ clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))
232
+ cl1 = clahe.apply(gray)
233
+ preprocessed_images.append(cv2.cvtColor(cl1, cv2.COLOR_GRAY2RGB))
234
+
235
+ # Thresholded
236
+ _, thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
237
+ preprocessed_images.append(cv2.cvtColor(thresh, cv2.COLOR_GRAY2RGB))
238
+
239
+ all_texts = []
240
+
241
+ for proc_img in preprocessed_images:
242
+ try:
243
+ results = ocr_reader.readtext(proc_img, detail=1, paragraph=True)
244
+ for (bbox, text, confidence) in results:
245
+ if confidence > 0.3 and len(text.strip()) > 1:
246
+ all_texts.append((text.strip(), confidence))
247
+ except:
248
+ continue
249
+
250
+ # Remove duplicates and low confidence texts
251
+ unique_texts = []
252
+ seen = set()
253
+ for text, conf in sorted(all_texts, key=lambda x: x[1], reverse=True):
254
+ text_clean = re.sub(r'\W+', '', text.lower())
255
+ if text_clean not in seen and len(text_clean) > 1:
256
+ unique_texts.append(text)
257
+ seen.add(text_clean)
258
+
259
+ return " | ".join(unique_texts[:5]) # Limit to top 5 unique texts
260
+
261
+ except Exception as e:
262
+ print(f"OCR error: {e}")
263
+ return ""
264
 
265
+ # === Enhanced CLIP classification ===
266
+ def precompute_embeddings():
267
+ """Precompute embeddings for all label categories."""
268
+ global _product_embeddings, _brand_embeddings, _character_embeddings, _detail_embeddings
269
+
270
+ product_prompts = [f"a photo of a {label}" for label in COMPREHENSIVE_PRODUCT_LABELS]
271
+ brand_prompts = [f"a {brand} product" for brand in COMPREHENSIVE_BRAND_LABELS]
272
+ character_prompts = [f"a photo of a {label}" for label in CHARACTER_LABELS]
273
+ detail_prompts = [f"an image showing {label}" for label in DETAIL_LABELS]
274
+
275
+ with torch.inference_mode():
276
+ _product_embeddings = clip_model.encode_text(open_clip.tokenize(product_prompts).to(device))
277
+ _product_embeddings = _product_embeddings / _product_embeddings.norm(dim=-1, keepdim=True)
278
+
279
+ _brand_embeddings = clip_model.encode_text(open_clip.tokenize(brand_prompts).to(device))
280
+ _brand_embeddings = _brand_embeddings / _brand_embeddings.norm(dim=-1, keepdim=True)
281
+
282
+ _character_embeddings = clip_model.encode_text(open_clip.tokenize(character_prompts).to(device))
283
+ _character_embeddings = _character_embeddings / _character_embeddings.norm(dim=-1, keepdim=True)
284
+
285
+ _detail_embeddings = clip_model.encode_text(open_clip.tokenize(detail_prompts).to(device))
286
+ _detail_embeddings = _detail_embeddings / _detail_embeddings.norm(dim=-1, keepdim=True)
287
+
288
+ def enhanced_zero_shot_classification(image: Image.Image):
289
+ """Enhanced zero-shot classification across multiple categories."""
290
  with torch.inference_mode():
291
  img_tensor = clip_preprocess(image).unsqueeze(0).to(device)
292
  img_emb = clip_model.encode_image(img_tensor)
293
  img_emb = img_emb / img_emb.norm(dim=-1, keepdim=True)
294
+
295
+ results = {}
296
+
297
+ # Product classification
298
+ product_logits = 100.0 * img_emb @ _product_embeddings.T
299
+ product_probs = torch.softmax(product_logits, dim=-1)
300
+ top_products = torch.topk(product_probs, 3, dim=-1)
301
+ results['products'] = [(COMPREHENSIVE_PRODUCT_LABELS[idx], float(prob))
302
+ for prob, idx in zip(top_products.values[0], top_products.indices[0])]
303
+
304
+ # Brand classification
305
+ brand_logits = 100.0 * img_emb @ _brand_embeddings.T
306
+ brand_probs = torch.softmax(brand_logits, dim=-1)
307
+ top_brands = torch.topk(brand_probs, 3, dim=-1)
308
+ results['brands'] = [(COMPREHENSIVE_BRAND_LABELS[idx], float(prob))
309
+ for prob, idx in zip(top_brands.values[0], top_brands.indices[0])]
310
+
311
+ # Character classification
312
+ character_logits = 100.0 * img_emb @ _character_embeddings.T
313
+ character_probs = torch.softmax(character_logits, dim=-1)
314
+ top_characters = torch.topk(character_probs, 3, dim=-1)
315
+ results['characters'] = [(CHARACTER_LABELS[idx], float(prob))
316
+ for prob, idx in zip(top_characters.values[0], top_characters.indices[0])]
317
+
318
+ # Detail classification
319
+ detail_logits = 100.0 * img_emb @ _detail_embeddings.T
320
+ detail_probs = torch.softmax(detail_logits, dim=-1)
321
+ top_details = torch.topk(detail_probs, 5, dim=-1)
322
+ results['details'] = [(DETAIL_LABELS[idx], float(prob))
323
+ for prob, idx in zip(top_details.values[0], top_details.indices[0])]
324
+
325
+ return results
326
+
327
+ # === Enhanced BLIP captioning ===
328
+ def get_enhanced_caption(image: Image.Image):
329
+ """Generate multiple captions with different prompts and combine insights."""
330
+ prompts = [
331
+ None, # Unconditional
332
+ "Describe this product in detail including its brand, type, colors, and any visible text or logos.",
333
+ "What are the key visual features and characteristics of this item?",
334
+ "Provide a comprehensive description of this object including materials, colors, and design elements."
335
+ ]
336
+
337
+ captions = []
338
+ for prompt in prompts:
339
+ try:
340
+ if prompt is None:
341
+ inputs = blip_processor(image, return_tensors="pt").to(device)
342
+ else:
343
+ inputs = blip_processor(image, text=prompt, return_tensors="pt").to(device)
344
+
345
+ with torch.inference_mode():
346
+ out = blip_model.generate(
347
+ **inputs,
348
+ max_new_tokens=80,
349
+ num_beams=5,
350
+ do_sample=True,
351
+ temperature=0.7,
352
+ early_stopping=True,
353
+ no_repeat_ngram_size=3
354
+ )
355
+
356
+ caption = blip_processor.decode(out[0], skip_special_tokens=True).strip()
357
+ if prompt and prompt.lower() in caption.lower():
358
+ continue # Skip if caption just repeats the prompt
359
+ captions.append(caption)
360
+
361
+ except Exception as e:
362
+ print(f"Caption generation error: {e}")
363
+ continue
364
+
365
+ # Return the longest, most detailed caption
366
+ return max(captions, key=len) if captions else "Unable to generate caption"
367
+
368
+ # === Comprehensive analysis pipeline ===
369
+ def comprehensive_image_analysis(image: Image.Image):
370
+ """Main analysis pipeline combining all techniques."""
371
+ # Resize for efficiency
372
+ if max(image.size) > 1024:
373
  image = image.copy()
374
+ image.thumbnail((1024, 1024), Image.Resampling.LANCZOS)
375
+
376
+ analysis = {}
377
+
378
+ # Enhanced captioning
379
+ analysis['caption'] = get_enhanced_caption(image)
380
+
381
+ # VQA analysis
382
+ analysis['vqa'] = analyze_with_vqa(image)
383
+
384
+ # Zero-shot classification
385
+ analysis['classification'] = enhanced_zero_shot_classification(image)
386
+
387
+ # OCR analysis
388
+ analysis['ocr_text'] = extract_comprehensive_ocr(image)
389
+
390
+ # Color analysis
391
+ analysis['colors'] = extract_dominant_colors(image)
392
+
393
+ return analysis
394
+
395
+ # === Alt-text generation ===
396
+ def build_comprehensive_alt_text(analysis):
397
+ """Build detailed alt-text from comprehensive analysis."""
 
 
 
 
 
 
 
 
 
398
  parts = []
399
+
400
+ # Main caption
401
+ if analysis.get('caption'):
402
+ parts.append(f"Description: {analysis['caption']}")
403
+
404
+ # Product type (from classification)
405
+ products = analysis.get('classification', {}).get('products', [])
406
+ if products and products[0][1] > 0.3: # Confidence threshold
407
+ parts.append(f"Product type: {products[0][0]}")
408
+
409
+ # Brand (from classification and VQA)
410
+ brands = analysis.get('classification', {}).get('brands', [])
411
+ brand_vqa = analysis.get('vqa', {}).get('brand is visible in this image', '')
412
+
413
+ brand_text = ""
414
+ if brands and brands[0][1] > 0.25 and brands[0][0] not in ['unknown', 'generic', 'no brand visible']:
415
+ brand_text = brands[0][0]
416
+ elif brand_vqa and len(brand_vqa) > 2 and brand_vqa.lower() not in ['no', 'none', 'unknown']:
417
+ brand_text = brand_vqa
418
+
419
+ if brand_text:
420
+ parts.append(f"Brand: {brand_text}")
421
+
422
+ # Characters/People
423
+ characters = analysis.get('classification', {}).get('characters', [])
424
+ if characters and characters[0][1] > 0.4:
425
+ parts.append(f"Features: {characters[0][0]}")
426
+
427
+ # Colors
428
+ colors = analysis.get('colors', [])
429
+ if colors:
430
+ color_text = ", ".join(colors[:3]) # Top 3 colors
431
+ parts.append(f"Dominant colors: {color_text}")
432
+
433
+ # OCR text
434
+ ocr_text = analysis.get('ocr_text', '')
435
  if ocr_text:
436
+ parts.append(f"Visible text: \"{ocr_text}\"")
437
+
438
+ # Key details
439
+ details = analysis.get('classification', {}).get('details', [])
440
+ high_conf_details = [detail for detail, conf in details if conf > 0.4]
441
+ if high_conf_details:
442
+ parts.append(f"Visual elements: {', '.join(high_conf_details[:3])}")
443
+
444
+ # VQA insights
445
+ vqa = analysis.get('vqa', {})
446
+ material = vqa.get('material does this product appear to be made of', '')
447
+ if material and len(material) > 2:
448
+ parts.append(f"Material: {material}")
449
+
450
+ background = vqa.get('background of this image', '')
451
+ if background and len(background) > 5 and background.lower() not in ['white', 'plain', 'simple']:
452
+ parts.append(f"Background: {background}")
453
+
454
+ # Combine all parts
455
+ alt_text = "; ".join(parts)
456
+
457
+ # Ensure proper ending
458
+ if not alt_text.endswith('.'):
459
+ alt_text += '.'
460
+
461
+ return alt_text
462
+
463
+ # === Initialize embeddings ===
464
+ print("Precomputing embeddings...")
465
+ precompute_embeddings()
466
+ print("Embeddings ready!")
467
+
468
+ # === Processing functions ===
 
 
469
  def handle_single_image(file):
470
+ """Process a single image file."""
471
  try:
472
  image = Image.open(file.name).convert("RGB")
473
  except Exception as e:
474
  return f"Failed to open image: {e}"
475
+
476
+ try:
477
+ analysis = comprehensive_image_analysis(image)
478
+ alt_text = build_comprehensive_alt_text(analysis)
479
+ return alt_text
480
+ except Exception as e:
481
+ return f"Analysis failed: {e}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
482
 
483
  def preview_images_grid(files):
484
+ """Generate preview grid for uploaded images."""
485
  if not files:
486
  return []
487
  previews = []
488
+ for file in files[:12]: # Limit preview
489
  try:
490
  img = Image.open(file.name).convert("RGB")
491
+ img.thumbnail((200, 200), Image.Resampling.LANCZOS)
492
  previews.append(img)
493
  except:
494
  continue
495
  return previews
496
 
497
+ def process_images(files):
498
+ """Process multiple images with controlled concurrency."""
499
  if not files:
500
  return "No images uploaded."
501
+
502
  results = []
503
+ # Use limited concurrency to avoid GPU memory issues
504
+ with ThreadPoolExecutor(max_workers=1) as executor: # Sequential processing for stability
505
+ futures = {executor.submit(handle_single_image, f): idx for idx, f in enumerate(files[:8])}
506
+
507
  for fut in as_completed(futures):
508
  idx = futures[fut]
509
  try:
510
+ alt_text = fut.result()
511
+ results.append(f"**Image {idx+1}:**\n{alt_text}\n")
512
  except Exception as e:
513
+ results.append(f"**Image {idx+1}:** Failed to process - {e}\n")
514
+
515
+ # Sort by original order
516
+ results_sorted = sorted(results, key=lambda s: int(re.search(r"Image (\d+):", s).group(1)))
517
+ return "\n".join(results_sorted)
518
 
519
+ # === Gradio Interface ===
520
  with gr.Blocks(
521
+ title="πŸ” Advanced Product Alt-Text Generator",
522
  theme=gr.themes.Soft(),
523
  css="""
524
+ .gradio-container { max-width: 1400px !important; }
525
+ .description-output {
526
+ font-family: 'Georgia', serif;
527
+ line-height: 1.6;
528
+ font-size: 14px;
529
+ }
530
+ .upload-section { border: 2px dashed #ccc; border-radius: 10px; padding: 20px; }
531
  """
532
  ) as demo:
533
+
534
+ gr.Markdown("""
535
+ # πŸ” Advanced Product Alt-Text Generator
536
+
537
+ **Enhanced with comprehensive analysis capabilities:**
538
+ - βœ… **Multi-model approach**: BLIP captioning + VQA + CLIP classification
539
+ - βœ… **Extensive object recognition**: 100+ product types, 50+ brands
540
+ - βœ… **Character & person detection**: Identifies people, mascots, characters
541
+ - βœ… **Advanced OCR**: Multi-language text extraction with preprocessing
542
+ - βœ… **Color analysis**: Dominant color extraction using K-means clustering
543
+ - βœ… **Visual question answering**: Asks specific questions about image content
544
+ - βœ… **Material & texture recognition**: Identifies materials and surface properties
545
+
546
+ Upload product images to generate detailed, accurate alt-text descriptions.
547
+ """)
548
+
549
  with gr.Row():
550
+ with gr.Column(scale=1, elem_classes=["upload-section"]):
551
  file_input = gr.File(
552
+ label="πŸ“€ Upload Product Images",
553
  file_count="multiple",
554
  file_types=["image"],
555
+ height=200,
556
  )
557
+ generate_btn = gr.Button(
558
+ "πŸš€ Generate Detailed Alt-Text",
559
+ variant="primary",
560
+ size="lg"
561
+ )
562
+
563
+ gr.Markdown("""
564
+ **Supported formats:** JPG, PNG, WebP, BMP
565
+ **Max images:** 8 per batch
566
+ **Recommended:** High-quality, well-lit product photos
567
+ """)
568
+
569
  with gr.Column(scale=2):
570
  gallery = gr.Gallery(
571
+ label="πŸ“Έ Image Preview",
572
  columns=4,
573
  rows=2,
574
+ height=350,
575
  object_fit="contain"
576
  )
577
+
578
  output = gr.Textbox(
579
+ label="πŸ“ Generated Alt-Text Descriptions",
580
+ lines=20,
581
  show_copy_button=True,
582
  elem_classes=["description-output"],
583
+ placeholder="Detailed alt-text descriptions will appear here after processing...",
584
+ max_lines=30
585
+ )
586
+
587
+ # Event handlers
588
+ file_input.change(
589
+ fn=preview_images_grid,
590
+ inputs=file_input,
591
+ outputs=gallery,
592
+ show_progress="hidden"
593
  )
594
+
595
+ generate_btn.click(
596
+ fn=process_images,
597
+ inputs=file_input,
598
+ outputs=output,
599
+ show_progress=True
 
 
 
600
  )
601
+
602
+ gr.Markdown("""
603
+ ---
604
+ **πŸ”§ Technical Features:**
605
+ - **Enhanced BLIP**: Uses large model with multiple prompting strategies
606
+ - **Comprehensive CLIP**: 200+ labels across products, brands, characters, details
607
+ - **Smart OCR**: Multi-preprocessing approaches for better text recognition
608
+ - **Color Intelligence**: Scientific color naming with clustering analysis
609
+ - **VQA Integration**: Asks targeted questions for specific details
610
+ - **Robust Processing**: Error handling and fallbacks for reliable results
611
+
612
+ **πŸ’‘ Best Results:** Use clear, well-lit images with visible branding and text.
613
+ """)
614
 
615
  if __name__ == "__main__":
616
+ demo.launch(
617
+ server_name="0.0.0.0",
618
+ server_port=7860,
619
+ share=False,
620
+ show_error=True
621
+ )