Update app.py
Browse files
app.py
CHANGED
|
@@ -8,37 +8,58 @@ import numpy as np
|
|
| 8 |
import re
|
| 9 |
from collections import Counter
|
| 10 |
from transformers import BlipProcessor, BlipForConditionalGeneration
|
|
|
|
| 11 |
|
| 12 |
# === Device setup ===
|
| 13 |
-
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 14 |
-
|
| 15 |
-
# === Model initialization ===
|
| 16 |
-
ocr_reader = easyocr.Reader(["en"], gpu=torch.cuda.is_available())
|
| 17 |
-
|
| 18 |
-
clip_model, _, clip_preprocess = open_clip.create_model_and_transforms("ViT-B-32", pretrained="openai")
|
| 19 |
-
clip_model.eval()
|
| 20 |
-
if device == "cuda":
|
| 21 |
-
clip_model = clip_model.cuda()
|
| 22 |
|
|
|
|
|
|
|
| 23 |
blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
|
| 24 |
blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
|
| 25 |
blip_model.to(device).eval()
|
| 26 |
-
if device == "cuda":
|
| 27 |
blip_model = blip_model.half()
|
| 28 |
-
torch.backends.cudnn.benchmark = True
|
| 29 |
|
| 30 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
DEFAULT_PRODUCT_LABELS = [
|
| 32 |
"trucker hat", "baseball cap", "cap", "board game box", "product box", "doll",
|
| 33 |
"brush marker", "paint marker", "marker", "card game box", "bottle", "package",
|
| 34 |
-
"headphones", "smartphone", "laptop", "watch", "shoe", "sneakers", "book", "skincare bottle"
|
|
|
|
| 35 |
]
|
| 36 |
DEFAULT_BRAND_LABELS = [
|
| 37 |
"Molotow", "Flash Point", "Evergreen", "Hasbro", "Mattel", "Ravensburger",
|
| 38 |
-
"Fantasy Flight", "Asmodee", "Days of Wonder", "CMON"
|
| 39 |
]
|
| 40 |
|
| 41 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
NAMED_COLORS = {
|
| 43 |
"white": (255, 255, 255),
|
| 44 |
"black": (0, 0, 0),
|
|
@@ -53,57 +74,37 @@ NAMED_COLORS = {
|
|
| 53 |
"pink": (255, 192, 203),
|
| 54 |
"purple": (128, 0, 128),
|
| 55 |
}
|
|
|
|
|
|
|
| 56 |
|
| 57 |
def nearest_color_name(rgb):
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
d = (r - cr)**2 + (g - cg)**2 + (b - cb)**2
|
| 63 |
-
if d < best_dist:
|
| 64 |
-
best_dist = d
|
| 65 |
-
best = name
|
| 66 |
-
return best
|
| 67 |
-
|
| 68 |
-
# === Core helpers ===
|
| 69 |
-
|
| 70 |
-
def zero_shot(image: Image.Image, candidates):
|
| 71 |
-
img_input = clip_preprocess(image).unsqueeze(0)
|
| 72 |
-
if device == "cuda":
|
| 73 |
-
img_input = img_input.cuda()
|
| 74 |
-
text_tokens = open_clip.tokenize(candidates)
|
| 75 |
-
if device == "cuda":
|
| 76 |
-
text_tokens = text_tokens.cuda()
|
| 77 |
-
with torch.no_grad():
|
| 78 |
-
image_emb = clip_model.encode_image(img_input)
|
| 79 |
-
text_emb = clip_model.encode_text(text_tokens)
|
| 80 |
-
image_emb = image_emb / image_emb.norm(dim=-1, keepdim=True)
|
| 81 |
-
text_emb = text_emb / text_emb.norm(dim=-1, keepdim=True)
|
| 82 |
-
logits = 100.0 * image_emb @ text_emb.T
|
| 83 |
-
probs = logits.softmax(dim=-1)
|
| 84 |
-
best_idx = probs.argmax(dim=-1).item()
|
| 85 |
-
return candidates[best_idx], probs[0, best_idx].item()
|
| 86 |
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
return
|
| 105 |
|
| 106 |
def extract_and_clean_ocr(image: Image.Image):
|
|
|
|
|
|
|
| 107 |
arr = np.array(image)
|
| 108 |
try:
|
| 109 |
data = ocr_reader.readtext(arr, detail=1)
|
|
@@ -135,19 +136,46 @@ def extract_and_clean_ocr(image: Image.Image):
|
|
| 135 |
for k, v in replacements.items():
|
| 136 |
raw = raw.replace(k, v)
|
| 137 |
|
| 138 |
-
# Capitalize first letter of sentence-like
|
| 139 |
return raw.strip().capitalize()
|
| 140 |
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
|
| 152 |
def dominant_color_of_image(image: Image.Image):
|
| 153 |
arr = np.array(image).reshape(-1, 3)
|
|
@@ -163,178 +191,120 @@ def dominant_color_for_object(image: Image.Image, product_type: str):
|
|
| 163 |
return None
|
| 164 |
return name
|
| 165 |
|
| 166 |
-
|
| 167 |
-
prompt = "Describe only the background of this image, excluding the main object."
|
| 168 |
-
caption = get_blip_caption(image, prompt=prompt)
|
| 169 |
-
if "describe only the background" in caption.lower():
|
| 170 |
-
return ""
|
| 171 |
-
return caption
|
| 172 |
|
| 173 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
gray = cv2.cvtColor(np.array(pil), cv2.COLOR_RGB2GRAY)
|
| 181 |
-
return cv2.Laplacian(gray, cv2.CV_64F).var()
|
| 182 |
-
front = image.crop((int(0.1 * w), int(0.3 * h), int(0.6 * w), int(0.8 * h)))
|
| 183 |
-
side = image.crop((int(0.6 * w), int(0.3 * h), int(0.95 * w), int(0.8 * h)))
|
| 184 |
-
front_var = lap_var(front)
|
| 185 |
-
side_var = lap_var(side)
|
| 186 |
-
if front_var < side_var * 0.8:
|
| 187 |
-
parts.append("with solid front panel and mesh sides for ventilation")
|
| 188 |
-
if ocr_text:
|
| 189 |
-
logo_text = ocr_text.strip()
|
| 190 |
-
if len(logo_text) == 1:
|
| 191 |
-
parts.append(f'rounded white logo featuring "{logo_text}" on front')
|
| 192 |
-
else:
|
| 193 |
-
parts.append(f'logo text: "{logo_text}"')
|
| 194 |
-
else:
|
| 195 |
-
parts.append('rounded white logo with stylized "M" on front')
|
| 196 |
-
return "; ".join(parts)
|
| 197 |
-
|
| 198 |
-
def describe_marker(image: Image.Image, ocr_text: str, fallback: str):
|
| 199 |
-
parts = []
|
| 200 |
-
parts.append("brush marker")
|
| 201 |
-
w, h = image.size
|
| 202 |
-
cap_region = image.crop((int(0.4 * w), 0, int(0.6 * w), int(0.2 * h)))
|
| 203 |
-
body_region = image.crop((int(0.3 * w), int(0.25 * h), int(0.7 * w), int(0.75 * h)))
|
| 204 |
-
def avg_color(pil):
|
| 205 |
-
arr = np.array(pil).reshape(-1, 3)
|
| 206 |
-
return tuple(arr.mean(axis=0).astype(int).tolist())
|
| 207 |
-
cap_rgb = avg_color(cap_region)
|
| 208 |
-
body_rgb = avg_color(body_region)
|
| 209 |
-
cap_name = nearest_color_name(cap_rgb)
|
| 210 |
-
body_name = nearest_color_name(body_rgb)
|
| 211 |
-
if cap_name:
|
| 212 |
-
parts.append(f"{cap_name} cap/tip")
|
| 213 |
-
if body_name:
|
| 214 |
-
parts.append(f"{body_name} body")
|
| 215 |
-
if ocr_text:
|
| 216 |
-
parts.append(f'Text on image: "{ocr_text}"')
|
| 217 |
-
return "; ".join(parts)
|
| 218 |
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
if ocr_text:
|
| 223 |
-
parts.append(f'Title/text: "{ocr_text}"')
|
| 224 |
-
if "firefighter" in fallback.lower() or "flame" in fallback.lower():
|
| 225 |
-
parts.append("featuring a firefighter amid flames")
|
| 226 |
-
return "; ".join(parts)
|
| 227 |
|
| 228 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 229 |
parts = []
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
if
|
| 237 |
-
parts.append(f"
|
| 238 |
-
|
| 239 |
-
parts.append("brown hair")
|
| 240 |
if ocr_text:
|
| 241 |
-
parts.append(f'Text
|
| 242 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
|
| 244 |
-
|
|
|
|
|
|
|
|
|
|
| 245 |
|
| 246 |
-
|
| 247 |
-
image = Image.open(image_path).convert("RGB")
|
| 248 |
-
if max(image.size) > 800:
|
| 249 |
-
image.thumbnail((800, 800), Image.Resampling.LANCZOS)
|
| 250 |
-
ocr_text = extract_and_clean_ocr(image)
|
| 251 |
-
fallback_caption = get_blip_caption(image)
|
| 252 |
|
| 253 |
-
|
| 254 |
combined_lower = (fallback_caption + " " + (ocr_text or "")).lower()
|
| 255 |
if "flash point" in combined_lower or "legacy of flame" in combined_lower:
|
| 256 |
-
parts = ['Flash Point: Legacy of Flame board game box cover']
|
| 257 |
-
parts.append('featuring a firefighter amid flames')
|
| 258 |
if "thrilling" in (ocr_text or "").lower() and "heroism" in (ocr_text or "").lower():
|
| 259 |
parts.append('tagline “A thrilling game of heroism and teamwork, where every choice matters.”')
|
| 260 |
elif ocr_text:
|
| 261 |
parts.append(f'tagline "{ocr_text}"')
|
| 262 |
-
# background is usually white for this cover; only add if scenic
|
| 263 |
if not is_solid_background(image):
|
| 264 |
-
bg =
|
| 265 |
if bg:
|
| 266 |
parts.append(f"Background: {bg}")
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
if ocr_text and b.lower() in ocr_text.lower():
|
| 273 |
-
brand = b
|
| 274 |
-
break
|
| 275 |
-
if not brand:
|
| 276 |
-
candidate_brand, score = zero_shot(image, DEFAULT_BRAND_LABELS + ["unknown"])
|
| 277 |
-
if candidate_brand != "unknown" and score > 0.2:
|
| 278 |
-
brand = candidate_brand
|
| 279 |
-
|
| 280 |
-
# Product type inference
|
| 281 |
-
product_type, prod_score = zero_shot(image, DEFAULT_PRODUCT_LABELS)
|
| 282 |
-
if (not product_type or prod_score < 0.2) and "hat" in fallback_caption.lower():
|
| 283 |
-
product_type = "trucker hat"
|
| 284 |
-
if (not product_type or prod_score < 0.2) and "marker" in fallback_caption.lower():
|
| 285 |
-
product_type = "brush marker"
|
| 286 |
-
if (not product_type or prod_score < 0.2) and "box" in fallback_caption.lower():
|
| 287 |
-
product_type = "product box"
|
| 288 |
-
|
| 289 |
-
# Background description
|
| 290 |
-
background_note = ""
|
| 291 |
-
if not is_solid_background(image):
|
| 292 |
-
bg_desc = describe_background_only_safe(image)
|
| 293 |
-
if bg_desc:
|
| 294 |
-
background_note = f" Background: {bg_desc}."
|
| 295 |
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
parts.append(f"in {color}")
|
| 324 |
-
if ocr_text and not any(ocr_text.lower() in p.lower() for p in parts):
|
| 325 |
-
parts.append(f'Text on image: "{ocr_text}"')
|
| 326 |
-
if fallback_caption and not any(fallback_caption.lower().split()[0] in p.lower() for p in parts):
|
| 327 |
-
parts.append(fallback_caption)
|
| 328 |
|
| 329 |
-
alt = "; ".join(parts).strip()
|
| 330 |
-
if background_note:
|
| 331 |
-
alt += background_note
|
| 332 |
-
alt = re.sub(r'\s+;', ';', alt)
|
| 333 |
-
if not alt.endswith("."):
|
| 334 |
-
alt += "."
|
| 335 |
return alt
|
| 336 |
|
| 337 |
-
# === Gradio UI ===
|
| 338 |
|
| 339 |
def preview_images_grid(files):
|
| 340 |
if not files:
|
|
@@ -353,13 +323,19 @@ def process_images(files, style):
|
|
| 353 |
if not files:
|
| 354 |
return "No images uploaded."
|
| 355 |
results = []
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 363 |
|
| 364 |
with gr.Blocks(
|
| 365 |
title="🛍️ Generalized Product Alt-text Generator",
|
|
|
|
| 8 |
import re
|
| 9 |
from collections import Counter
|
| 10 |
from transformers import BlipProcessor, BlipForConditionalGeneration
|
| 11 |
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 12 |
|
| 13 |
# === Device setup ===
|
| 14 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 15 |
+
torch.backends.cudnn.benchmark = True # speeds up variable-size inputs
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
+
# === Model initialization (once) ===
|
| 18 |
+
# BLIP for captions
|
| 19 |
blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
|
| 20 |
blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
|
| 21 |
blip_model.to(device).eval()
|
| 22 |
+
if device.type == "cuda":
|
| 23 |
blip_model = blip_model.half()
|
|
|
|
| 24 |
|
| 25 |
+
# CLIP for zero-shot classification
|
| 26 |
+
clip_model, _, clip_preprocess = open_clip.create_model_and_transforms("ViT-B-32", pretrained="openai")
|
| 27 |
+
clip_model.to(device).eval()
|
| 28 |
+
if device.type == "cuda":
|
| 29 |
+
clip_model = clip_model.half()
|
| 30 |
+
|
| 31 |
+
# OCR reader (English only, can add other langs)
|
| 32 |
+
ocr_reader = easyocr.Reader(["en"], gpu=(device.type == "cuda"))
|
| 33 |
+
|
| 34 |
+
# === Default label sets (extendable) ===
|
| 35 |
DEFAULT_PRODUCT_LABELS = [
|
| 36 |
"trucker hat", "baseball cap", "cap", "board game box", "product box", "doll",
|
| 37 |
"brush marker", "paint marker", "marker", "card game box", "bottle", "package",
|
| 38 |
+
"headphones", "smartphone", "laptop", "watch", "shoe", "sneakers", "book", "skincare bottle",
|
| 39 |
+
"cosmetic jar", "perfume bottle", "toy figure", "sporting equipment", "water bottle"
|
| 40 |
]
|
| 41 |
DEFAULT_BRAND_LABELS = [
|
| 42 |
"Molotow", "Flash Point", "Evergreen", "Hasbro", "Mattel", "Ravensburger",
|
| 43 |
+
"Fantasy Flight", "Asmodee", "Days of Wonder", "CMON", "Nike", "Adidas", "Sony", "Samsung"
|
| 44 |
]
|
| 45 |
|
| 46 |
+
# Precompute text embeddings for zero-shot (with simple prompt templates)
|
| 47 |
+
def embed_texts(texts):
|
| 48 |
+
with torch.inference_mode():
|
| 49 |
+
tokens = open_clip.tokenize(texts).to(device)
|
| 50 |
+
emb = clip_model.encode_text(tokens)
|
| 51 |
+
emb = emb / emb.norm(dim=-1, keepdim=True)
|
| 52 |
+
return emb # shape (len(texts), dim)
|
| 53 |
+
|
| 54 |
+
_product_prompts = [f"a photo of a {p}" for p in DEFAULT_PRODUCT_LABELS]
|
| 55 |
+
_brand_prompts = [f"a product from {b}" for b in DEFAULT_BRAND_LABELS] + ["unknown"]
|
| 56 |
+
|
| 57 |
+
_product_label_emb = embed_texts(_product_prompts)
|
| 58 |
+
_brand_label_emb = embed_texts(_brand_prompts)
|
| 59 |
+
_product_labels_extended = DEFAULT_PRODUCT_LABELS.copy()
|
| 60 |
+
_brand_labels_extended = DEFAULT_BRAND_LABELS + ["unknown"]
|
| 61 |
+
|
| 62 |
+
# === Named colors for approximation (vectorized) ===
|
| 63 |
NAMED_COLORS = {
|
| 64 |
"white": (255, 255, 255),
|
| 65 |
"black": (0, 0, 0),
|
|
|
|
| 74 |
"pink": (255, 192, 203),
|
| 75 |
"purple": (128, 0, 128),
|
| 76 |
}
|
| 77 |
+
_color_names = np.array(list(NAMED_COLORS.keys()))
|
| 78 |
+
_color_values = np.array(list(NAMED_COLORS.values())) # (N,3)
|
| 79 |
|
| 80 |
def nearest_color_name(rgb):
|
| 81 |
+
rgb_arr = np.array(rgb, dtype=int)
|
| 82 |
+
dists = np.sum((_color_values - rgb_arr) ** 2, axis=1)
|
| 83 |
+
idx = int(np.argmin(dists))
|
| 84 |
+
return _color_names[idx]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
|
| 86 |
+
# === Utility / gating helpers ===
|
| 87 |
+
|
| 88 |
+
def is_solid_background(image: Image.Image, color_threshold=0.85, edge_threshold=0.03):
|
| 89 |
+
small = cv2.resize(np.array(image), (100, 100), interpolation=cv2.INTER_AREA)
|
| 90 |
+
pixels = small.reshape(-1, 3)
|
| 91 |
+
tup = [tuple(p) for p in pixels]
|
| 92 |
+
cnt = Counter(tup)
|
| 93 |
+
dominant_frac = cnt.most_common(1)[0][1] / len(tup)
|
| 94 |
+
gray = cv2.cvtColor(small, cv2.COLOR_RGB2GRAY)
|
| 95 |
+
edges = cv2.Canny(gray, 100, 200)
|
| 96 |
+
edge_density = np.mean(edges > 0)
|
| 97 |
+
return dominant_frac >= color_threshold and edge_density <= edge_threshold
|
| 98 |
+
|
| 99 |
+
def should_run_ocr(image: Image.Image, edge_density_threshold=0.01):
|
| 100 |
+
gray = cv2.cvtColor(np.array(image.resize((200, 200))), cv2.COLOR_RGB2GRAY)
|
| 101 |
+
edges = cv2.Canny(gray, 100, 200)
|
| 102 |
+
edge_density = np.mean(edges > 0)
|
| 103 |
+
return edge_density >= edge_density_threshold # low edge density likely no text/logo
|
| 104 |
|
| 105 |
def extract_and_clean_ocr(image: Image.Image):
|
| 106 |
+
if not should_run_ocr(image):
|
| 107 |
+
return ""
|
| 108 |
arr = np.array(image)
|
| 109 |
try:
|
| 110 |
data = ocr_reader.readtext(arr, detail=1)
|
|
|
|
| 136 |
for k, v in replacements.items():
|
| 137 |
raw = raw.replace(k, v)
|
| 138 |
|
|
|
|
| 139 |
return raw.strip().capitalize()
|
| 140 |
|
| 141 |
+
# === Captioning with BLIP (ensemble of prompts) ===
|
| 142 |
+
|
| 143 |
+
def get_blip_caption(image: Image.Image, prompt: str = None):
|
| 144 |
+
query = prompt or "Describe the product in the image including its type, visible text/logo, brand if any, and key visual features."
|
| 145 |
+
inputs = blip_processor(image, text=query, return_tensors="pt").to(device)
|
| 146 |
+
with torch.inference_mode():
|
| 147 |
+
out = blip_model.generate(
|
| 148 |
+
**inputs,
|
| 149 |
+
max_new_tokens=60 if prompt is None else 40,
|
| 150 |
+
num_beams=4,
|
| 151 |
+
do_sample=False,
|
| 152 |
+
early_stopping=True,
|
| 153 |
+
no_repeat_ngram_size=2
|
| 154 |
+
)
|
| 155 |
+
caption = blip_processor.decode(out[0], skip_special_tokens=True)
|
| 156 |
+
caption = re.sub(r'^(the|a|an)\s+', '', caption, flags=re.IGNORECASE).strip()
|
| 157 |
+
return caption.capitalize()
|
| 158 |
+
|
| 159 |
+
def get_background_caption(image: Image.Image):
|
| 160 |
+
prompt = "Describe only the background of this image, excluding the main object."
|
| 161 |
+
caption = get_blip_caption(image, prompt=prompt)
|
| 162 |
+
if "describe only the background" in caption.lower():
|
| 163 |
+
return ""
|
| 164 |
+
return caption
|
| 165 |
+
|
| 166 |
+
# === CLIP zero-shot inference using cached embeddings ===
|
| 167 |
+
|
| 168 |
+
def zero_shot_clip(image: Image.Image, label_embs, labels):
|
| 169 |
+
with torch.inference_mode():
|
| 170 |
+
img_tensor = clip_preprocess(image).unsqueeze(0).to(device)
|
| 171 |
+
img_emb = clip_model.encode_image(img_tensor)
|
| 172 |
+
img_emb = img_emb / img_emb.norm(dim=-1, keepdim=True)
|
| 173 |
+
logits = 100.0 * img_emb @ label_embs.T # (1, num_labels)
|
| 174 |
+
probs = torch.softmax(logits, dim=-1)
|
| 175 |
+
best_idx = int(probs.argmax())
|
| 176 |
+
return labels[best_idx], float(probs[0, best_idx])
|
| 177 |
+
|
| 178 |
+
# === Color extraction ===
|
| 179 |
|
| 180 |
def dominant_color_of_image(image: Image.Image):
|
| 181 |
arr = np.array(image).reshape(-1, 3)
|
|
|
|
| 191 |
return None
|
| 192 |
return name
|
| 193 |
|
| 194 |
+
# === Structured inference combining signals ===
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 195 |
|
| 196 |
+
def infer_product_info(image: Image.Image):
|
| 197 |
+
# Resize to reasonable size early for speed
|
| 198 |
+
if max(image.size) > 800:
|
| 199 |
+
image = image.copy()
|
| 200 |
+
image.thumbnail((800, 800), Image.Resampling.LANCZOS)
|
| 201 |
|
| 202 |
+
# Captions (ensemble: base and alternate phrasing)
|
| 203 |
+
caption1 = get_blip_caption(image)
|
| 204 |
+
caption2 = get_blip_caption(image, prompt="Provide a detailed description of the product, including its type, brand cues, visible text/logo, and distinguishing features.")
|
| 205 |
+
# Choose longer or merge heuristically
|
| 206 |
+
caption = caption2 if len(caption2) > len(caption1) else caption1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
|
| 208 |
+
# Zero-shot product type and brand
|
| 209 |
+
product_type, prod_conf = zero_shot_clip(image, _product_label_emb, _product_labels_extended)
|
| 210 |
+
brand, brand_conf = zero_shot_clip(image, _brand_label_emb, _brand_labels_extended)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
|
| 212 |
+
# Dominant color (skip boxes)
|
| 213 |
+
color = dominant_color_for_object(image, product_type or "")
|
| 214 |
+
|
| 215 |
+
# Background description if not solid
|
| 216 |
+
background_desc = ""
|
| 217 |
+
if not is_solid_background(image):
|
| 218 |
+
background_desc = get_background_caption(image)
|
| 219 |
+
|
| 220 |
+
return {
|
| 221 |
+
"brand": brand if brand != "unknown" else None,
|
| 222 |
+
"brand_confidence": brand_conf,
|
| 223 |
+
"product_type": product_type,
|
| 224 |
+
"product_confidence": prod_conf,
|
| 225 |
+
"caption": caption,
|
| 226 |
+
"dominant_color": color,
|
| 227 |
+
"background": background_desc,
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
# === Alt-text builder ===
|
| 231 |
+
|
| 232 |
+
def build_alt_text(structured, ocr_text=""):
|
| 233 |
parts = []
|
| 234 |
+
# Brand and product
|
| 235 |
+
if structured["brand"]:
|
| 236 |
+
parts.append(f"{structured['brand']}")
|
| 237 |
+
if structured["product_type"]:
|
| 238 |
+
parts.append(f"{structured['product_type']}")
|
| 239 |
+
# Color
|
| 240 |
+
if structured["dominant_color"]:
|
| 241 |
+
parts.append(f"in {structured['dominant_color']}")
|
| 242 |
+
# OCR
|
|
|
|
| 243 |
if ocr_text:
|
| 244 |
+
parts.append(f'Text visible: "{ocr_text}"')
|
| 245 |
+
# Caption
|
| 246 |
+
if structured["caption"]:
|
| 247 |
+
parts.append(f"Caption: {structured['caption']}")
|
| 248 |
+
# Background
|
| 249 |
+
if structured["background"]:
|
| 250 |
+
parts.append(f"Background: {structured['background']}")
|
| 251 |
|
| 252 |
+
alt = "; ".join(parts).strip()
|
| 253 |
+
if not alt.endswith("."):
|
| 254 |
+
alt += "."
|
| 255 |
+
return alt
|
| 256 |
|
| 257 |
+
# === Flash Point special override ===
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 258 |
|
| 259 |
+
def flash_point_override(image: Image.Image, ocr_text: str, fallback_caption: str):
|
| 260 |
combined_lower = (fallback_caption + " " + (ocr_text or "")).lower()
|
| 261 |
if "flash point" in combined_lower or "legacy of flame" in combined_lower:
|
| 262 |
+
parts = ['Flash Point: Legacy of Flame board game box cover', 'featuring a firefighter amid flames']
|
|
|
|
| 263 |
if "thrilling" in (ocr_text or "").lower() and "heroism" in (ocr_text or "").lower():
|
| 264 |
parts.append('tagline “A thrilling game of heroism and teamwork, where every choice matters.”')
|
| 265 |
elif ocr_text:
|
| 266 |
parts.append(f'tagline "{ocr_text}"')
|
|
|
|
| 267 |
if not is_solid_background(image):
|
| 268 |
+
bg = get_background_caption(image)
|
| 269 |
if bg:
|
| 270 |
parts.append(f"Background: {bg}")
|
| 271 |
+
alt = "; ".join(parts)
|
| 272 |
+
if not alt.endswith("."):
|
| 273 |
+
alt += "."
|
| 274 |
+
return alt
|
| 275 |
+
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 276 |
|
| 277 |
+
# === Image processing pipeline (single image) ===
|
| 278 |
+
|
| 279 |
+
def handle_single_image(file):
|
| 280 |
+
try:
|
| 281 |
+
image = Image.open(file.name).convert("RGB")
|
| 282 |
+
except Exception as e:
|
| 283 |
+
return f"Failed to open image: {e}"
|
| 284 |
+
|
| 285 |
+
if max(image.size) > 800:
|
| 286 |
+
image.thumbnail((800, 800), Image.Resampling.LANCZOS)
|
| 287 |
+
|
| 288 |
+
# OCR
|
| 289 |
+
ocr_text = extract_and_clean_ocr(image)
|
| 290 |
+
|
| 291 |
+
# BLIP fallback caption (for override)
|
| 292 |
+
fallback_caption = get_blip_caption(image)
|
| 293 |
+
|
| 294 |
+
# Flash Point override check
|
| 295 |
+
fp = flash_point_override(image, ocr_text, fallback_caption)
|
| 296 |
+
if fp:
|
| 297 |
+
return fp
|
| 298 |
+
|
| 299 |
+
# Structured inference
|
| 300 |
+
structured = infer_product_info(image)
|
| 301 |
+
|
| 302 |
+
# Build alt-text
|
| 303 |
+
alt = build_alt_text(structured, ocr_text=ocr_text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 304 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 305 |
return alt
|
| 306 |
|
| 307 |
+
# === Gradio UI helpers ===
|
| 308 |
|
| 309 |
def preview_images_grid(files):
|
| 310 |
if not files:
|
|
|
|
| 323 |
if not files:
|
| 324 |
return "No images uploaded."
|
| 325 |
results = []
|
| 326 |
+
# Controlled concurrency: 2 workers to avoid GPU contention
|
| 327 |
+
with ThreadPoolExecutor(max_workers=2) as executor:
|
| 328 |
+
futures = {executor.submit(handle_single_image, f): idx for idx, f in enumerate(files[:10])}
|
| 329 |
+
for fut in as_completed(futures):
|
| 330 |
+
idx = futures[fut]
|
| 331 |
+
try:
|
| 332 |
+
alt = fut.result()
|
| 333 |
+
results.append(f"Image {idx+1}: {alt}")
|
| 334 |
+
except Exception as e:
|
| 335 |
+
results.append(f"Image {idx+1}: Failed to describe ({e})")
|
| 336 |
+
# Sort by original index to preserve order
|
| 337 |
+
results_sorted = sorted(results, key=lambda s: int(re.match(r"Image (\d+):", s).group(1)))
|
| 338 |
+
return "\n\n".join(results_sorted)
|
| 339 |
|
| 340 |
with gr.Blocks(
|
| 341 |
title="🛍️ Generalized Product Alt-text Generator",
|