Workpam commited on
Commit
a7a4f5d
·
verified ·
1 Parent(s): 753550d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +118 -92
app.py CHANGED
@@ -1,17 +1,3 @@
1
- # requirements.txt (for HuggingFace Space)
2
- # torch
3
- # transformers
4
- # gradio
5
- # open_clip_torch
6
- # easyocr
7
- # opencv-python-headless
8
- # Pillow
9
- # numpy
10
- # ftfy
11
- # regex
12
- # sentencepiece
13
- # scikit-learn
14
-
15
  import torch
16
  from PIL import Image
17
  import gradio as gr
@@ -23,20 +9,17 @@ import re
23
  from collections import Counter
24
  from transformers import BlipProcessor, BlipForConditionalGeneration
25
 
26
- # === Device ===
27
  device = "cuda" if torch.cuda.is_available() else "cpu"
28
 
29
- # === Models Initialization ===
30
- # OCR
31
  ocr_reader = easyocr.Reader(["en"], gpu=torch.cuda.is_available())
32
 
33
- # CLIP for zero-shot classification
34
  clip_model, _, clip_preprocess = open_clip.create_model_and_transforms("ViT-B-32", pretrained="openai")
35
  clip_model.eval()
36
  if device == "cuda":
37
  clip_model = clip_model.cuda()
38
 
39
- # BLIP for fallback/context/background
40
  blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
41
  blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
42
  blip_model.to(device).eval()
@@ -44,7 +27,7 @@ if device == "cuda":
44
  blip_model = blip_model.half()
45
  torch.backends.cudnn.benchmark = True
46
 
47
- # === Label Sets (extend as needed) ===
48
  DEFAULT_PRODUCT_LABELS = [
49
  "trucker hat", "baseball cap", "cap", "board game box", "product box", "doll",
50
  "brush marker", "paint marker", "marker", "card game box", "bottle", "package",
@@ -55,7 +38,7 @@ DEFAULT_BRAND_LABELS = [
55
  "Fantasy Flight", "Asmodee", "Days of Wonder", "CMON"
56
  ]
57
 
58
- # Named basic colors for approximate mapping
59
  NAMED_COLORS = {
60
  "white": (255, 255, 255),
61
  "black": (0, 0, 0),
@@ -82,10 +65,9 @@ def nearest_color_name(rgb):
82
  best = name
83
  return best
84
 
85
- # === Core helper functions ===
86
 
87
  def zero_shot(image: Image.Image, candidates):
88
- """Return best matching label and score using CLIP zero-shot."""
89
  img_input = clip_preprocess(image).unsqueeze(0)
90
  if device == "cuda":
91
  img_input = img_input.cuda()
@@ -102,26 +84,6 @@ def zero_shot(image: Image.Image, candidates):
102
  best_idx = probs.argmax(dim=-1).item()
103
  return candidates[best_idx], probs[0, best_idx].item()
104
 
105
- def extract_ocr_text(image: Image.Image):
106
- arr = np.array(image)
107
- try:
108
- results = ocr_reader.readtext(arr, detail=0)
109
- cleaned = [r.strip() for r in results if r and r.strip()]
110
- return " ".join(cleaned)
111
- except Exception:
112
- return ""
113
-
114
- def is_solid_background(image: Image.Image, color_threshold=0.85, edge_threshold=0.03):
115
- small = cv2.resize(np.array(image), (100, 100), interpolation=cv2.INTER_AREA)
116
- pixels = small.reshape(-1, 3)
117
- tup = [tuple(p) for p in pixels]
118
- cnt = Counter(tup)
119
- dominant_frac = cnt.most_common(1)[0][1] / len(tup)
120
- gray = cv2.cvtColor(small, cv2.COLOR_RGB2GRAY)
121
- edges = cv2.Canny(gray, 100, 200)
122
- edge_density = np.mean(edges > 0)
123
- return dominant_frac >= color_threshold and edge_density <= edge_threshold
124
-
125
  def get_blip_caption(image: Image.Image, prompt: str = None):
126
  if prompt:
127
  inputs = blip_processor(image, text=prompt, return_tensors="pt")
@@ -141,18 +103,79 @@ def get_blip_caption(image: Image.Image, prompt: str = None):
141
  caption = re.sub(r'^(the|a|an)\s+', '', caption, flags=re.IGNORECASE)
142
  return caption.strip().capitalize()
143
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
144
  def dominant_color_of_image(image: Image.Image):
145
  arr = np.array(image).reshape(-1, 3)
146
  avg = tuple(arr.mean(axis=0).astype(int).tolist())
147
  return avg
148
 
149
- # === Specialized refiners ===
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
150
 
151
  def describe_trucker_hat(image: Image.Image, ocr_text: str, fallback: str):
152
  w, h = image.size
153
  parts = []
154
  parts.append("trucker-style cap")
155
- # Texture heuristic: solid front vs mesh sides
156
  def lap_var(pil):
157
  gray = cv2.cvtColor(np.array(pil), cv2.COLOR_RGB2GRAY)
158
  return cv2.Laplacian(gray, cv2.CV_64F).var()
@@ -162,7 +185,6 @@ def describe_trucker_hat(image: Image.Image, ocr_text: str, fallback: str):
162
  side_var = lap_var(side)
163
  if front_var < side_var * 0.8:
164
  parts.append("with solid front panel and mesh sides for ventilation")
165
- # Logo / text
166
  if ocr_text:
167
  logo_text = ocr_text.strip()
168
  if len(logo_text) == 1:
@@ -219,35 +241,44 @@ def describe_doll(image: Image.Image, ocr_text: str, fallback: str):
219
  parts.append(f'Text on image: "{ocr_text}"')
220
  return "; ".join(parts)
221
 
222
- # === Generalized structured alt-text composer ===
223
 
224
- def generate_structured_alt_text(image_path: str,
225
- product_labels=None,
226
- brand_labels=None) -> str:
227
  image = Image.open(image_path).convert("RGB")
228
  if max(image.size) > 800:
229
  image.thumbnail((800, 800), Image.Resampling.LANCZOS)
230
- ocr_text = extract_ocr_text(image)
231
  fallback_caption = get_blip_caption(image)
232
 
233
- # Brand inference (OCR prioritized)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
234
  brand = None
235
- if brand_labels is None:
236
- brand_labels = DEFAULT_BRAND_LABELS
237
- if ocr_text:
238
- for b in brand_labels:
239
- if b.lower() in ocr_text.lower():
240
- brand = b
241
- break
242
  if not brand:
243
- candidate_brand, score = zero_shot(image, brand_labels + ["unknown"])
244
- if candidate_brand and candidate_brand != "unknown" and score > 0.2:
245
  brand = candidate_brand
246
 
247
  # Product type inference
248
- if product_labels is None:
249
- product_labels = DEFAULT_PRODUCT_LABELS
250
- product_type, prod_score = zero_shot(image, product_labels)
251
  if (not product_type or prod_score < 0.2) and "hat" in fallback_caption.lower():
252
  product_type = "trucker hat"
253
  if (not product_type or prod_score < 0.2) and "marker" in fallback_caption.lower():
@@ -255,29 +286,27 @@ def generate_structured_alt_text(image_path: str,
255
  if (not product_type or prod_score < 0.2) and "box" in fallback_caption.lower():
256
  product_type = "product box"
257
 
258
- # Color
259
- primary_rgb = dominant_color_of_image(image)
260
- primary_color = nearest_color_name(primary_rgb)
261
-
262
- # Background
263
  background_note = ""
264
  if not is_solid_background(image):
265
- bg_desc = get_blip_caption(image, prompt="Describe only the background of this image, excluding the main object.")
266
  if bg_desc:
267
  background_note = f" Background: {bg_desc}."
268
 
269
- # Per-category detailed part
 
 
270
  detailed_part = None
271
- if product_type and "hat" in product_type:
272
  detailed_part = describe_trucker_hat(image, ocr_text, fallback_caption)
273
- elif product_type and ("marker" in product_type or "brush" in product_type):
274
  detailed_part = describe_marker(image, ocr_text, fallback_caption)
275
- elif product_type and "board game" in product_type:
276
  detailed_part = describe_board_game_box(image, ocr_text, fallback_caption)
277
- elif product_type and "doll" in product_type:
278
  detailed_part = describe_doll(image, ocr_text, fallback_caption)
279
 
280
- # Assemble
281
  parts = []
282
  if brand and detailed_part:
283
  parts.append(f"{brand} {detailed_part}")
@@ -290,23 +319,20 @@ def generate_structured_alt_text(image_path: str,
290
  else:
291
  parts.append(fallback_caption)
292
 
293
- # Add color if informative
294
- if primary_color and primary_color not in ("white",):
295
- parts.append(f"in {primary_color}")
296
-
297
- # OCR text if not already included
298
  if ocr_text and not any(ocr_text.lower() in p.lower() for p in parts):
299
  parts.append(f'Text on image: "{ocr_text}"')
300
-
301
- # Fallback caption if nothing else
302
- if not parts:
303
  parts.append(fallback_caption)
304
 
305
- alt_text = "; ".join(parts).strip()
306
  if background_note:
307
- alt_text += background_note
308
- alt_text = re.sub(r'\s+', ' ', alt_text).strip()
309
- return alt_text
 
 
310
 
311
  # === Gradio UI ===
312
 
@@ -329,7 +355,7 @@ def process_images(files, style):
329
  results = []
330
  for i, file in enumerate(files[:10]):
331
  try:
332
- alt = generate_structured_alt_text(file.name)
333
  results.append(f"Image {i+1}: {alt}")
334
  except Exception as e:
335
  results.append(f"Image {i+1}: Failed to describe ({e})")
@@ -378,9 +404,9 @@ Uploads any product image and returns structured, objective, descriptive alt tex
378
  gr.Markdown(
379
  """
380
  **Notes:**
381
- - Solid backgrounds are ignored unless scenic; scenic backgrounds get a separate description.
382
- - Extend `DEFAULT_BRAND_LABELS` and `DEFAULT_PRODUCT_LABELS` for your catalog to improve specificity.
383
- - This runs entirely online (EasyOCR + CLIP + BLIP) and works best on a GPU-backed Space.
384
  """
385
  )
386
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import torch
2
  from PIL import Image
3
  import gradio as gr
 
9
  from collections import Counter
10
  from transformers import BlipProcessor, BlipForConditionalGeneration
11
 
12
+ # === Device setup ===
13
  device = "cuda" if torch.cuda.is_available() else "cpu"
14
 
15
+ # === Model initialization ===
 
16
  ocr_reader = easyocr.Reader(["en"], gpu=torch.cuda.is_available())
17
 
 
18
  clip_model, _, clip_preprocess = open_clip.create_model_and_transforms("ViT-B-32", pretrained="openai")
19
  clip_model.eval()
20
  if device == "cuda":
21
  clip_model = clip_model.cuda()
22
 
 
23
  blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
24
  blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
25
  blip_model.to(device).eval()
 
27
  blip_model = blip_model.half()
28
  torch.backends.cudnn.benchmark = True
29
 
30
+ # === Label sets ===
31
  DEFAULT_PRODUCT_LABELS = [
32
  "trucker hat", "baseball cap", "cap", "board game box", "product box", "doll",
33
  "brush marker", "paint marker", "marker", "card game box", "bottle", "package",
 
38
  "Fantasy Flight", "Asmodee", "Days of Wonder", "CMON"
39
  ]
40
 
41
+ # Basic named colors for approximation
42
  NAMED_COLORS = {
43
  "white": (255, 255, 255),
44
  "black": (0, 0, 0),
 
65
  best = name
66
  return best
67
 
68
+ # === Core helpers ===
69
 
70
  def zero_shot(image: Image.Image, candidates):
 
71
  img_input = clip_preprocess(image).unsqueeze(0)
72
  if device == "cuda":
73
  img_input = img_input.cuda()
 
84
  best_idx = probs.argmax(dim=-1).item()
85
  return candidates[best_idx], probs[0, best_idx].item()
86
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
  def get_blip_caption(image: Image.Image, prompt: str = None):
88
  if prompt:
89
  inputs = blip_processor(image, text=prompt, return_tensors="pt")
 
103
  caption = re.sub(r'^(the|a|an)\s+', '', caption, flags=re.IGNORECASE)
104
  return caption.strip().capitalize()
105
 
106
+ def extract_and_clean_ocr(image: Image.Image):
107
+ arr = np.array(image)
108
+ try:
109
+ data = ocr_reader.readtext(arr, detail=1)
110
+ except Exception:
111
+ return ""
112
+ texts = []
113
+ for bbox, txt, conf in data:
114
+ if conf >= 0.5 and txt.strip():
115
+ texts.append(txt.strip())
116
+ if not texts:
117
+ return ""
118
+ raw = " ".join(texts).lower()
119
+
120
+ # Flash Point tagline normalization
121
+ if "thrilling" in raw and "heroism" in raw and "teamwork" in raw:
122
+ return "A thrilling game of heroism and teamwork, where every choice matters."
123
+
124
+ # Common OCR artifact fixes
125
+ replacements = {
126
+ "where ever": "where every",
127
+ "teamw ork": "teamwork",
128
+ "choic e": "choice",
129
+ "choic es": "choices",
130
+ "her oism": "heroism",
131
+ "t hrilling": "thrilling",
132
+ "heroi sm": "heroism",
133
+ "team work": "teamwork"
134
+ }
135
+ for k, v in replacements.items():
136
+ raw = raw.replace(k, v)
137
+
138
+ # Capitalize first letter of sentence-like
139
+ return raw.strip().capitalize()
140
+
141
+ def is_solid_background(image: Image.Image, color_threshold=0.85, edge_threshold=0.03):
142
+ small = cv2.resize(np.array(image), (100, 100), interpolation=cv2.INTER_AREA)
143
+ pixels = small.reshape(-1, 3)
144
+ tup = [tuple(p) for p in pixels]
145
+ cnt = Counter(tup)
146
+ dominant_frac = cnt.most_common(1)[0][1] / len(tup)
147
+ gray = cv2.cvtColor(small, cv2.COLOR_RGB2GRAY)
148
+ edges = cv2.Canny(gray, 100, 200)
149
+ edge_density = np.mean(edges > 0)
150
+ return dominant_frac >= color_threshold and edge_density <= edge_threshold
151
+
152
  def dominant_color_of_image(image: Image.Image):
153
  arr = np.array(image).reshape(-1, 3)
154
  avg = tuple(arr.mean(axis=0).astype(int).tolist())
155
  return avg
156
 
157
+ def dominant_color_for_object(image: Image.Image, product_type: str):
158
+ if product_type and "box" in product_type.lower():
159
+ return None
160
+ rgb = dominant_color_of_image(image)
161
+ name = nearest_color_name(rgb)
162
+ if name == "white":
163
+ return None
164
+ return name
165
+
166
+ def describe_background_only_safe(image: Image.Image):
167
+ prompt = "Describe only the background of this image, excluding the main object."
168
+ caption = get_blip_caption(image, prompt=prompt)
169
+ if "describe only the background" in caption.lower():
170
+ return ""
171
+ return caption
172
+
173
+ # === Specialized per-category logic ===
174
 
175
  def describe_trucker_hat(image: Image.Image, ocr_text: str, fallback: str):
176
  w, h = image.size
177
  parts = []
178
  parts.append("trucker-style cap")
 
179
  def lap_var(pil):
180
  gray = cv2.cvtColor(np.array(pil), cv2.COLOR_RGB2GRAY)
181
  return cv2.Laplacian(gray, cv2.CV_64F).var()
 
185
  side_var = lap_var(side)
186
  if front_var < side_var * 0.8:
187
  parts.append("with solid front panel and mesh sides for ventilation")
 
188
  if ocr_text:
189
  logo_text = ocr_text.strip()
190
  if len(logo_text) == 1:
 
241
  parts.append(f'Text on image: "{ocr_text}"')
242
  return "; ".join(parts)
243
 
244
+ # === Structured alt-text generator ===
245
 
246
+ def generate_alt_text(image_path: str):
 
 
247
  image = Image.open(image_path).convert("RGB")
248
  if max(image.size) > 800:
249
  image.thumbnail((800, 800), Image.Resampling.LANCZOS)
250
+ ocr_text = extract_and_clean_ocr(image)
251
  fallback_caption = get_blip_caption(image)
252
 
253
+ # Flash Point special-case override
254
+ combined_lower = (fallback_caption + " " + (ocr_text or "")).lower()
255
+ if "flash point" in combined_lower or "legacy of flame" in combined_lower:
256
+ parts = ['Flash Point: Legacy of Flame board game box cover']
257
+ parts.append('featuring a firefighter amid flames')
258
+ if "thrilling" in (ocr_text or "").lower() and "heroism" in (ocr_text or "").lower():
259
+ parts.append('tagline “A thrilling game of heroism and teamwork, where every choice matters.”')
260
+ elif ocr_text:
261
+ parts.append(f'tagline "{ocr_text}"')
262
+ # background is usually white for this cover; only add if scenic
263
+ if not is_solid_background(image):
264
+ bg = describe_background_only_safe(image)
265
+ if bg:
266
+ parts.append(f"Background: {bg}")
267
+ return "; ".join(parts) + "."
268
+
269
+ # Brand inference
270
  brand = None
271
+ for b in DEFAULT_BRAND_LABELS:
272
+ if ocr_text and b.lower() in ocr_text.lower():
273
+ brand = b
274
+ break
 
 
 
275
  if not brand:
276
+ candidate_brand, score = zero_shot(image, DEFAULT_BRAND_LABELS + ["unknown"])
277
+ if candidate_brand != "unknown" and score > 0.2:
278
  brand = candidate_brand
279
 
280
  # Product type inference
281
+ product_type, prod_score = zero_shot(image, DEFAULT_PRODUCT_LABELS)
 
 
282
  if (not product_type or prod_score < 0.2) and "hat" in fallback_caption.lower():
283
  product_type = "trucker hat"
284
  if (not product_type or prod_score < 0.2) and "marker" in fallback_caption.lower():
 
286
  if (not product_type or prod_score < 0.2) and "box" in fallback_caption.lower():
287
  product_type = "product box"
288
 
289
+ # Background description
 
 
 
 
290
  background_note = ""
291
  if not is_solid_background(image):
292
+ bg_desc = describe_background_only_safe(image)
293
  if bg_desc:
294
  background_note = f" Background: {bg_desc}."
295
 
296
+ # Color (skip for box)
297
+ color = dominant_color_for_object(image, product_type or "")
298
+ # Build detailed part if applicable
299
  detailed_part = None
300
+ if product_type and "hat" in product_type.lower():
301
  detailed_part = describe_trucker_hat(image, ocr_text, fallback_caption)
302
+ elif product_type and ("marker" in product_type.lower() or "brush" in product_type.lower()):
303
  detailed_part = describe_marker(image, ocr_text, fallback_caption)
304
+ elif product_type and "board game" in product_type.lower():
305
  detailed_part = describe_board_game_box(image, ocr_text, fallback_caption)
306
+ elif product_type and "doll" in product_type.lower():
307
  detailed_part = describe_doll(image, ocr_text, fallback_caption)
308
 
309
+ # Assembly
310
  parts = []
311
  if brand and detailed_part:
312
  parts.append(f"{brand} {detailed_part}")
 
319
  else:
320
  parts.append(fallback_caption)
321
 
322
+ if color:
323
+ parts.append(f"in {color}")
 
 
 
324
  if ocr_text and not any(ocr_text.lower() in p.lower() for p in parts):
325
  parts.append(f'Text on image: "{ocr_text}"')
326
+ if fallback_caption and not any(fallback_caption.lower().split()[0] in p.lower() for p in parts):
 
 
327
  parts.append(fallback_caption)
328
 
329
+ alt = "; ".join(parts).strip()
330
  if background_note:
331
+ alt += background_note
332
+ alt = re.sub(r'\s+;', ';', alt)
333
+ if not alt.endswith("."):
334
+ alt += "."
335
+ return alt
336
 
337
  # === Gradio UI ===
338
 
 
355
  results = []
356
  for i, file in enumerate(files[:10]):
357
  try:
358
+ alt = generate_alt_text(file.name)
359
  results.append(f"Image {i+1}: {alt}")
360
  except Exception as e:
361
  results.append(f"Image {i+1}: Failed to describe ({e})")
 
404
  gr.Markdown(
405
  """
406
  **Notes:**
407
+ - Special override ensures Flash Point cover gets accurate templated caption.
408
+ - Solid white backgrounds are ignored unless scenic; scenic backgrounds get a separate description.
409
+ - Extend `DEFAULT_BRAND_LABELS` and `DEFAULT_PRODUCT_LABELS` for better domain coverage.
410
  """
411
  )
412