Update app.py
Browse files
app.py
CHANGED
|
@@ -7,416 +7,615 @@ import cv2
|
|
| 7 |
import numpy as np
|
| 8 |
import re
|
| 9 |
from collections import Counter
|
| 10 |
-
from transformers import BlipProcessor, BlipForConditionalGeneration
|
| 11 |
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
# === Device setup ===
|
| 14 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 15 |
-
torch.backends.cudnn.benchmark = True
|
|
|
|
|
|
|
| 16 |
|
| 17 |
-
# === Model initialization
|
| 18 |
# BLIP for captions
|
| 19 |
-
blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-
|
| 20 |
-
blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-
|
| 21 |
blip_model.to(device).eval()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
if device.type == "cuda":
|
| 23 |
blip_model = blip_model.half()
|
|
|
|
| 24 |
|
| 25 |
# CLIP for zero-shot classification
|
| 26 |
-
clip_model, _, clip_preprocess = open_clip.create_model_and_transforms("ViT-
|
| 27 |
clip_model.to(device).eval()
|
| 28 |
if device.type == "cuda":
|
| 29 |
clip_model = clip_model.half()
|
| 30 |
|
| 31 |
-
# OCR reader
|
| 32 |
-
ocr_reader = easyocr.Reader(["en"], gpu=(device.type == "cuda"))
|
| 33 |
-
|
| 34 |
-
# ===
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
]
|
| 43 |
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
"
|
| 55 |
-
"
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
"
|
| 60 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
]
|
| 62 |
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
emb = emb / emb.norm(dim=-1, keepdim=True)
|
| 69 |
-
return emb # shape (len(texts), dim)
|
| 70 |
-
|
| 71 |
-
_product_prompts = [f"a photo of a {p}" for p in DEFAULT_PRODUCT_LABELS]
|
| 72 |
-
_brand_prompts = [f"a product from {b}" for b in DEFAULT_BRAND_LABELS] + ["unknown"]
|
| 73 |
-
|
| 74 |
-
_product_label_emb = embed_texts(_product_prompts)
|
| 75 |
-
_brand_label_emb = embed_texts(_brand_prompts)
|
| 76 |
-
_product_labels_extended = DEFAULT_PRODUCT_LABELS.copy()
|
| 77 |
-
_brand_labels_extended = DEFAULT_BRAND_LABELS + ["unknown"]
|
| 78 |
-
|
| 79 |
-
# === Named colors for approximation (vectorized) ===
|
| 80 |
-
NAMED_COLORS = {
|
| 81 |
-
"white": (255, 255, 255),
|
| 82 |
-
"black": (0, 0, 0),
|
| 83 |
-
"beige": (245, 245, 220),
|
| 84 |
-
"orange": (255, 165, 0),
|
| 85 |
-
"red": (220, 20, 60),
|
| 86 |
-
"blue": (30, 144, 255),
|
| 87 |
-
"green": (34, 139, 34),
|
| 88 |
-
"yellow": (255, 215, 0),
|
| 89 |
-
"brown": (139, 69, 19),
|
| 90 |
-
"gray": (128, 128, 128),
|
| 91 |
-
"pink": (255, 192, 203),
|
| 92 |
-
"purple": (128, 0, 128),
|
| 93 |
-
}
|
| 94 |
-
_color_names = np.array(list(NAMED_COLORS.keys()))
|
| 95 |
-
_color_values = np.array(list(NAMED_COLORS.values())) # (N,3)
|
| 96 |
-
|
| 97 |
-
def nearest_color_name(rgb):
|
| 98 |
-
rgb_arr = np.array(rgb, dtype=int)
|
| 99 |
-
dists = np.sum((_color_values - rgb_arr) ** 2, axis=1)
|
| 100 |
-
idx = int(np.argmin(dists))
|
| 101 |
-
return _color_names[idx]
|
| 102 |
-
|
| 103 |
-
# === Utility / gating helpers ===
|
| 104 |
-
|
| 105 |
-
def is_solid_background(image: Image.Image, color_threshold=0.85, edge_threshold=0.03):
|
| 106 |
-
small = cv2.resize(np.array(image), (100, 100), interpolation=cv2.INTER_AREA)
|
| 107 |
-
pixels = small.reshape(-1, 3)
|
| 108 |
-
tup = [tuple(p) for p in pixels]
|
| 109 |
-
cnt = Counter(tup)
|
| 110 |
-
dominant_frac = cnt.most_common(1)[0][1] / len(tup)
|
| 111 |
-
gray = cv2.cvtColor(small, cv2.COLOR_RGB2GRAY)
|
| 112 |
-
edges = cv2.Canny(gray, 100, 200)
|
| 113 |
-
edge_density = np.mean(edges > 0)
|
| 114 |
-
return dominant_frac >= color_threshold and edge_density <= edge_threshold
|
| 115 |
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
|
|
|
| 121 |
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
return ""
|
| 125 |
-
arr = np.array(image)
|
| 126 |
try:
|
| 127 |
-
|
| 128 |
-
except
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
|
|
|
|
|
|
| 163 |
with torch.inference_mode():
|
| 164 |
-
out =
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
early_stopping=True,
|
| 170 |
-
no_repeat_ngram_size=2
|
| 171 |
-
)
|
| 172 |
-
caption = blip_processor.decode(out[0], skip_special_tokens=True).strip()
|
| 173 |
-
return caption
|
| 174 |
-
|
| 175 |
-
base_prompt = prompt or "Describe the product in the image including its type, visible text/logo, brand if any, and key visual features."
|
| 176 |
-
caption = generate_with(base_prompt)
|
| 177 |
-
|
| 178 |
-
# If caption is too short or just repeats the instruction, retry with a different phrasing
|
| 179 |
-
if (
|
| 180 |
-
len(caption.split()) < 3
|
| 181 |
-
or base_prompt.lower() in caption.lower()
|
| 182 |
-
or re.search(r"describe the product", caption, flags=re.IGNORECASE)
|
| 183 |
-
):
|
| 184 |
-
alt_prompt = "Provide a concise descriptive caption of the product, its brand, and distinguishing visual attributes."
|
| 185 |
-
caption = generate_with(alt_prompt)
|
| 186 |
-
|
| 187 |
-
caption = re.sub(r'^(the|a|an)\s+', '', caption, flags=re.IGNORECASE).strip()
|
| 188 |
-
return caption.capitalize()
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
def get_background_caption(image: Image.Image):
|
| 192 |
-
prompt = "Describe only the background of this image, excluding the main object."
|
| 193 |
-
caption = get_blip_caption(image, prompt=prompt)
|
| 194 |
-
if "describe only the background" in caption.lower():
|
| 195 |
return ""
|
| 196 |
-
return caption
|
| 197 |
|
| 198 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 199 |
|
| 200 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
with torch.inference_mode():
|
| 202 |
img_tensor = clip_preprocess(image).unsqueeze(0).to(device)
|
| 203 |
img_emb = clip_model.encode_image(img_tensor)
|
| 204 |
img_emb = img_emb / img_emb.norm(dim=-1, keepdim=True)
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
image = image.copy()
|
| 232 |
-
image.thumbnail((
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
"product_confidence": prod_conf,
|
| 257 |
-
"caption": caption,
|
| 258 |
-
"dominant_color": color,
|
| 259 |
-
"background": background_desc,
|
| 260 |
-
}
|
| 261 |
-
|
| 262 |
-
# === Alt-text builder ===
|
| 263 |
-
|
| 264 |
-
def build_alt_text(structured, ocr_text=""):
|
| 265 |
parts = []
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
#
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 275 |
if ocr_text:
|
| 276 |
-
parts.append(f
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
if
|
| 282 |
-
parts.append(f"
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
# === Image processing pipeline (single image) ===
|
| 310 |
-
|
| 311 |
def handle_single_image(file):
|
|
|
|
| 312 |
try:
|
| 313 |
image = Image.open(file.name).convert("RGB")
|
| 314 |
except Exception as e:
|
| 315 |
return f"Failed to open image: {e}"
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
# BLIP fallback caption (for override)
|
| 324 |
-
fallback_caption = get_blip_caption(image)
|
| 325 |
-
|
| 326 |
-
# Flash Point override check
|
| 327 |
-
fp = flash_point_override(image, ocr_text, fallback_caption)
|
| 328 |
-
if fp:
|
| 329 |
-
return fp
|
| 330 |
-
|
| 331 |
-
# Structured inference
|
| 332 |
-
structured = infer_product_info(image)
|
| 333 |
-
|
| 334 |
-
# Build alt-text
|
| 335 |
-
alt = build_alt_text(structured, ocr_text=ocr_text)
|
| 336 |
-
|
| 337 |
-
return alt
|
| 338 |
-
|
| 339 |
-
# === Gradio UI helpers ===
|
| 340 |
|
| 341 |
def preview_images_grid(files):
|
|
|
|
| 342 |
if not files:
|
| 343 |
return []
|
| 344 |
previews = []
|
| 345 |
-
for file in files[:
|
| 346 |
try:
|
| 347 |
img = Image.open(file.name).convert("RGB")
|
| 348 |
-
img.thumbnail((
|
| 349 |
previews.append(img)
|
| 350 |
except:
|
| 351 |
continue
|
| 352 |
return previews
|
| 353 |
|
| 354 |
-
def process_images(files
|
|
|
|
| 355 |
if not files:
|
| 356 |
return "No images uploaded."
|
|
|
|
| 357 |
results = []
|
| 358 |
-
#
|
| 359 |
-
with ThreadPoolExecutor(max_workers=
|
| 360 |
-
futures = {executor.submit(handle_single_image, f): idx for idx, f in enumerate(files[:
|
|
|
|
| 361 |
for fut in as_completed(futures):
|
| 362 |
idx = futures[fut]
|
| 363 |
try:
|
| 364 |
-
|
| 365 |
-
results.append(f"Image {idx+1}:
|
| 366 |
except Exception as e:
|
| 367 |
-
results.append(f"Image {idx+1}: Failed to
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
|
|
|
| 371 |
|
|
|
|
| 372 |
with gr.Blocks(
|
| 373 |
-
title="
|
| 374 |
theme=gr.themes.Soft(),
|
| 375 |
css="""
|
| 376 |
-
.gradio-container { max-width:
|
| 377 |
-
.description-output {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 378 |
"""
|
| 379 |
) as demo:
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
#
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 386 |
with gr.Row():
|
| 387 |
-
with gr.Column(scale=1):
|
| 388 |
file_input = gr.File(
|
| 389 |
-
label="Upload
|
| 390 |
file_count="multiple",
|
| 391 |
file_types=["image"],
|
| 392 |
-
height=
|
| 393 |
)
|
| 394 |
-
generate_btn = gr.Button(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 395 |
with gr.Column(scale=2):
|
| 396 |
gallery = gr.Gallery(
|
| 397 |
-
label="Image Preview",
|
| 398 |
columns=4,
|
| 399 |
rows=2,
|
| 400 |
-
height=
|
| 401 |
object_fit="contain"
|
| 402 |
)
|
|
|
|
| 403 |
output = gr.Textbox(
|
| 404 |
-
label="Generated Alt
|
| 405 |
-
lines=
|
| 406 |
show_copy_button=True,
|
| 407 |
elem_classes=["description-output"],
|
| 408 |
-
placeholder="
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
)
|
| 410 |
-
|
| 411 |
-
generate_btn.click(
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
- Solid white backgrounds are ignored unless scenic; scenic backgrounds get a separate description.
|
| 417 |
-
- Extend `DEFAULT_BRAND_LABELS` and `DEFAULT_PRODUCT_LABELS` for better domain coverage.
|
| 418 |
-
"""
|
| 419 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 420 |
|
| 421 |
if __name__ == "__main__":
|
| 422 |
-
demo.launch(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
import numpy as np
|
| 8 |
import re
|
| 9 |
from collections import Counter
|
| 10 |
+
from transformers import BlipProcessor, BlipForConditionalGeneration, BlipProcessor, BlipForQuestionAnswering
|
| 11 |
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 12 |
+
from sklearn.cluster import KMeans
|
| 13 |
+
import webcolors
|
| 14 |
+
from difflib import get_close_matches
|
| 15 |
|
| 16 |
# === Device setup ===
|
| 17 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 18 |
+
torch.backends.cudnn.benchmark = True
|
| 19 |
+
|
| 20 |
+
print(f"Using device: {device}")
|
| 21 |
|
| 22 |
+
# === Model initialization ===
|
| 23 |
# BLIP for captions
|
| 24 |
+
blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large")
|
| 25 |
+
blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
|
| 26 |
blip_model.to(device).eval()
|
| 27 |
+
|
| 28 |
+
# BLIP for VQA (Visual Question Answering)
|
| 29 |
+
blip_vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
|
| 30 |
+
blip_vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
|
| 31 |
+
blip_vqa_model.to(device).eval()
|
| 32 |
+
|
| 33 |
if device.type == "cuda":
|
| 34 |
blip_model = blip_model.half()
|
| 35 |
+
blip_vqa_model = blip_vqa_model.half()
|
| 36 |
|
| 37 |
# CLIP for zero-shot classification
|
| 38 |
+
clip_model, _, clip_preprocess = open_clip.create_model_and_transforms("ViT-L-14", pretrained="openai")
|
| 39 |
clip_model.to(device).eval()
|
| 40 |
if device.type == "cuda":
|
| 41 |
clip_model = clip_model.half()
|
| 42 |
|
| 43 |
+
# OCR reader with multiple languages
|
| 44 |
+
ocr_reader = easyocr.Reader(["en", "es", "fr", "de", "it"], gpu=(device.type == "cuda"))
|
| 45 |
+
|
| 46 |
+
# === Expanded label sets ===
|
| 47 |
+
COMPREHENSIVE_PRODUCT_LABELS = [
|
| 48 |
+
# Accessories & Fashion
|
| 49 |
+
"trucker hat", "baseball cap", "beanie", "cap", "hat", "sunglasses", "watch", "jewelry",
|
| 50 |
+
"necklace", "bracelet", "ring", "earrings", "bag", "backpack", "purse", "wallet",
|
| 51 |
+
"belt", "scarf", "gloves", "shoes", "sneakers", "boots", "sandals", "heels",
|
| 52 |
+
|
| 53 |
+
# Electronics & Tech
|
| 54 |
+
"smartphone", "laptop", "tablet", "headphones", "earbuds", "speaker", "camera",
|
| 55 |
+
"gaming console", "controller", "keyboard", "mouse", "monitor", "TV", "remote",
|
| 56 |
+
"charger", "cable", "smartwatch", "fitness tracker", "drone", "robot",
|
| 57 |
+
|
| 58 |
+
# Toys & Games
|
| 59 |
+
"board game", "card game", "puzzle", "action figure", "doll", "toy car", "LEGO",
|
| 60 |
+
"building blocks", "stuffed animal", "teddy bear", "toy robot", "game controller",
|
| 61 |
+
"dice", "playing cards", "collectible card", "miniature figure", "model kit",
|
| 62 |
+
|
| 63 |
+
# Beauty & Personal Care
|
| 64 |
+
"perfume bottle", "cologne", "lipstick", "mascara", "foundation", "powder compact",
|
| 65 |
+
"nail polish", "skincare bottle", "lotion", "cream jar", "soap", "shampoo",
|
| 66 |
+
"conditioner", "toothbrush", "razor", "makeup brush", "mirror", "hair dryer",
|
| 67 |
+
|
| 68 |
+
# Food & Beverages
|
| 69 |
+
"water bottle", "soda can", "beer bottle", "wine bottle", "coffee mug", "tea cup",
|
| 70 |
+
"food package", "cereal box", "snack bag", "candy bar", "chocolate", "cookie package",
|
| 71 |
+
|
| 72 |
+
# Books & Media
|
| 73 |
+
"book", "magazine", "comic book", "DVD", "Blu-ray", "CD", "vinyl record",
|
| 74 |
+
"notebook", "journal", "planner", "calendar", "poster", "artwork", "photo frame",
|
| 75 |
+
|
| 76 |
+
# Sports & Fitness
|
| 77 |
+
"basketball", "football", "soccer ball", "tennis ball", "baseball", "golf ball",
|
| 78 |
+
"dumbbells", "yoga mat", "water bottle", "sports equipment", "helmet", "gloves",
|
| 79 |
+
|
| 80 |
+
# Home & Kitchen
|
| 81 |
+
"coffee maker", "blender", "toaster", "microwave", "pan", "pot", "plate", "bowl",
|
| 82 |
+
"cup", "glass", "utensils", "knife", "cutting board", "storage container",
|
| 83 |
+
|
| 84 |
+
# Art & Craft Supplies
|
| 85 |
+
"paint brush", "marker", "pen", "pencil", "crayon", "paint tube", "canvas",
|
| 86 |
+
"sketchbook", "ruler", "scissors", "glue", "tape", "stickers",
|
| 87 |
+
|
| 88 |
+
# Packaging & Containers
|
| 89 |
+
"box", "package", "container", "jar", "bottle", "can", "tube", "pouch", "bag"
|
| 90 |
]
|
| 91 |
|
| 92 |
+
COMPREHENSIVE_BRAND_LABELS = [
|
| 93 |
+
# Tech Brands
|
| 94 |
+
"Apple", "Samsung", "Google", "Microsoft", "Sony", "Nintendo", "PlayStation", "Xbox",
|
| 95 |
+
"HP", "Dell", "Lenovo", "ASUS", "Acer", "Canon", "Nikon", "GoPro", "DJI",
|
| 96 |
+
|
| 97 |
+
# Fashion & Accessories
|
| 98 |
+
"Nike", "Adidas", "Puma", "Under Armour", "Reebok", "Converse", "Vans", "New Balance",
|
| 99 |
+
"Ray-Ban", "Oakley", "Rolex", "Casio", "Fossil", "Michael Kors", "Coach", "Louis Vuitton",
|
| 100 |
+
|
| 101 |
+
# Beauty & Personal Care
|
| 102 |
+
"L'Oreal", "Maybelline", "MAC", "Sephora", "Clinique", "Estee Lauder", "Chanel",
|
| 103 |
+
"Dior", "YSL", "Urban Decay", "Too Faced", "NARS", "Fenty Beauty",
|
| 104 |
+
|
| 105 |
+
# Gaming & Toys
|
| 106 |
+
"Hasbro", "Mattel", "LEGO", "Ravensburger", "Fantasy Flight Games", "Asmodee",
|
| 107 |
+
"Days of Wonder", "CMON", "Wizards of the Coast", "PokΓ©mon", "Yu-Gi-Oh",
|
| 108 |
+
|
| 109 |
+
# Food & Beverages
|
| 110 |
+
"Coca-Cola", "Pepsi", "Starbucks", "Red Bull", "Monster", "Gatorade", "Powerade",
|
| 111 |
+
"Nestle", "Hershey's", "Mars", "Ferrero", "Lindt", "Godiva",
|
| 112 |
+
|
| 113 |
+
# Art & Craft
|
| 114 |
+
"Crayola", "Sharpie", "Prismacolor", "Winsor & Newton", "Molotow", "Posca",
|
| 115 |
+
"Staedtler", "Faber-Castell", "Sakura", "Tombow", "Pilot", "Uni-ball",
|
| 116 |
+
|
| 117 |
+
# Specialty/Generic
|
| 118 |
+
"unknown", "generic", "no brand visible"
|
| 119 |
]
|
| 120 |
|
| 121 |
+
CHARACTER_LABELS = [
|
| 122 |
+
"person", "man", "woman", "child", "baby", "elderly person", "athlete", "professional",
|
| 123 |
+
"cartoon character", "anime character", "superhero", "mascot", "celebrity", "model",
|
| 124 |
+
"fantasy character", "sci-fi character", "historical figure", "fictional character"
|
| 125 |
+
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
|
| 127 |
+
DETAIL_LABELS = [
|
| 128 |
+
"text", "logo", "brand name", "price tag", "barcode", "QR code", "warning label",
|
| 129 |
+
"ingredients list", "nutritional information", "instructions", "serial number",
|
| 130 |
+
"model number", "size information", "color variants", "pattern", "texture",
|
| 131 |
+
"metallic finish", "matte finish", "glossy finish", "transparent", "translucent"
|
| 132 |
+
]
|
| 133 |
|
| 134 |
+
# === Enhanced color detection ===
|
| 135 |
+
def get_color_name(rgb_tuple):
|
|
|
|
|
|
|
| 136 |
try:
|
| 137 |
+
closest_name = webcolors.rgb_to_name(rgb_tuple)
|
| 138 |
+
except ValueError:
|
| 139 |
+
# Find closest color
|
| 140 |
+
min_colours = {}
|
| 141 |
+
for key, name in webcolors.CSS3_HEX_TO_NAMES.items():
|
| 142 |
+
r_c, g_c, b_c = webcolors.hex_to_rgb(key)
|
| 143 |
+
rd = (r_c - rgb_tuple[0]) ** 2
|
| 144 |
+
gd = (g_c - rgb_tuple[1]) ** 2
|
| 145 |
+
bd = (b_c - rgb_tuple[2]) ** 2
|
| 146 |
+
min_colours[(rd + gd + bd)] = name
|
| 147 |
+
closest_name = min_colours[min(min_colours.keys())]
|
| 148 |
+
return closest_name
|
| 149 |
+
|
| 150 |
+
def extract_dominant_colors(image: Image.Image, n_colors=3):
|
| 151 |
+
"""Extract dominant colors using K-means clustering."""
|
| 152 |
+
img_array = np.array(image.resize((150, 150)))
|
| 153 |
+
img_array = img_array.reshape(-1, 3)
|
| 154 |
+
|
| 155 |
+
# Remove pure white and black pixels for better color analysis
|
| 156 |
+
mask = ~((img_array == [255, 255, 255]).all(axis=1) | (img_array == [0, 0, 0]).all(axis=1))
|
| 157 |
+
if mask.sum() > 0:
|
| 158 |
+
img_array = img_array[mask]
|
| 159 |
+
|
| 160 |
+
if len(img_array) < n_colors:
|
| 161 |
+
return []
|
| 162 |
+
|
| 163 |
+
kmeans = KMeans(n_clusters=n_colors, random_state=42, n_init=10)
|
| 164 |
+
kmeans.fit(img_array)
|
| 165 |
+
|
| 166 |
+
colors = []
|
| 167 |
+
for center in kmeans.cluster_centers_:
|
| 168 |
+
rgb = tuple(map(int, center))
|
| 169 |
+
try:
|
| 170 |
+
color_name = get_color_name(rgb)
|
| 171 |
+
colors.append(color_name)
|
| 172 |
+
except:
|
| 173 |
+
colors.append(f"RGB{rgb}")
|
| 174 |
+
|
| 175 |
+
return colors
|
| 176 |
|
| 177 |
+
# === Enhanced VQA for detailed analysis ===
|
| 178 |
+
def ask_visual_question(image: Image.Image, question: str):
|
| 179 |
+
"""Use BLIP VQA to ask specific questions about the image."""
|
| 180 |
+
try:
|
| 181 |
+
inputs = blip_vqa_processor(image, question, return_tensors="pt").to(device)
|
| 182 |
with torch.inference_mode():
|
| 183 |
+
out = blip_vqa_model.generate(**inputs, max_length=50, num_beams=4, early_stopping=True)
|
| 184 |
+
answer = blip_vqa_processor.decode(out[0], skip_special_tokens=True)
|
| 185 |
+
return answer.strip()
|
| 186 |
+
except Exception as e:
|
| 187 |
+
print(f"VQA error: {e}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 188 |
return ""
|
|
|
|
| 189 |
|
| 190 |
+
def analyze_with_vqa(image: Image.Image):
|
| 191 |
+
"""Analyze image using multiple VQA questions."""
|
| 192 |
+
questions = [
|
| 193 |
+
"What type of product is this?",
|
| 194 |
+
"What brand is visible in this image?",
|
| 195 |
+
"What colors are prominent in this image?",
|
| 196 |
+
"Is there any text visible in this image?",
|
| 197 |
+
"Are there any people or characters in this image?",
|
| 198 |
+
"What is the main object in this image?",
|
| 199 |
+
"What material does this product appear to be made of?",
|
| 200 |
+
"What is the background of this image?",
|
| 201 |
+
"Are there any logos or symbols visible?",
|
| 202 |
+
"What is the shape of the main object?"
|
| 203 |
+
]
|
| 204 |
+
|
| 205 |
+
results = {}
|
| 206 |
+
for question in questions:
|
| 207 |
+
answer = ask_visual_question(image, question)
|
| 208 |
+
if answer and len(answer) > 2: # Filter out very short/meaningless answers
|
| 209 |
+
key = question.lower().replace("what ", "").replace("is ", "").replace("are ", "").replace("?", "")
|
| 210 |
+
results[key] = answer
|
| 211 |
+
|
| 212 |
+
return results
|
| 213 |
+
|
| 214 |
+
# === Enhanced OCR with better text processing ===
|
| 215 |
+
def extract_comprehensive_ocr(image: Image.Image):
|
| 216 |
+
"""Enhanced OCR with better preprocessing and post-processing."""
|
| 217 |
+
try:
|
| 218 |
+
# Convert to numpy array
|
| 219 |
+
img_array = np.array(image)
|
| 220 |
+
|
| 221 |
+
# Preprocessing for better OCR
|
| 222 |
+
gray = cv2.cvtColor(img_array, cv2.COLOR_RGB2GRAY)
|
| 223 |
+
|
| 224 |
+
# Multiple preprocessing approaches
|
| 225 |
+
preprocessed_images = []
|
| 226 |
+
|
| 227 |
+
# Original
|
| 228 |
+
preprocessed_images.append(img_array)
|
| 229 |
+
|
| 230 |
+
# High contrast
|
| 231 |
+
clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))
|
| 232 |
+
cl1 = clahe.apply(gray)
|
| 233 |
+
preprocessed_images.append(cv2.cvtColor(cl1, cv2.COLOR_GRAY2RGB))
|
| 234 |
+
|
| 235 |
+
# Thresholded
|
| 236 |
+
_, thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
|
| 237 |
+
preprocessed_images.append(cv2.cvtColor(thresh, cv2.COLOR_GRAY2RGB))
|
| 238 |
+
|
| 239 |
+
all_texts = []
|
| 240 |
+
|
| 241 |
+
for proc_img in preprocessed_images:
|
| 242 |
+
try:
|
| 243 |
+
results = ocr_reader.readtext(proc_img, detail=1, paragraph=True)
|
| 244 |
+
for (bbox, text, confidence) in results:
|
| 245 |
+
if confidence > 0.3 and len(text.strip()) > 1:
|
| 246 |
+
all_texts.append((text.strip(), confidence))
|
| 247 |
+
except:
|
| 248 |
+
continue
|
| 249 |
+
|
| 250 |
+
# Remove duplicates and low confidence texts
|
| 251 |
+
unique_texts = []
|
| 252 |
+
seen = set()
|
| 253 |
+
for text, conf in sorted(all_texts, key=lambda x: x[1], reverse=True):
|
| 254 |
+
text_clean = re.sub(r'\W+', '', text.lower())
|
| 255 |
+
if text_clean not in seen and len(text_clean) > 1:
|
| 256 |
+
unique_texts.append(text)
|
| 257 |
+
seen.add(text_clean)
|
| 258 |
+
|
| 259 |
+
return " | ".join(unique_texts[:5]) # Limit to top 5 unique texts
|
| 260 |
+
|
| 261 |
+
except Exception as e:
|
| 262 |
+
print(f"OCR error: {e}")
|
| 263 |
+
return ""
|
| 264 |
|
| 265 |
+
# === Enhanced CLIP classification ===
|
| 266 |
+
def precompute_embeddings():
|
| 267 |
+
"""Precompute embeddings for all label categories."""
|
| 268 |
+
global _product_embeddings, _brand_embeddings, _character_embeddings, _detail_embeddings
|
| 269 |
+
|
| 270 |
+
product_prompts = [f"a photo of a {label}" for label in COMPREHENSIVE_PRODUCT_LABELS]
|
| 271 |
+
brand_prompts = [f"a {brand} product" for brand in COMPREHENSIVE_BRAND_LABELS]
|
| 272 |
+
character_prompts = [f"a photo of a {label}" for label in CHARACTER_LABELS]
|
| 273 |
+
detail_prompts = [f"an image showing {label}" for label in DETAIL_LABELS]
|
| 274 |
+
|
| 275 |
+
with torch.inference_mode():
|
| 276 |
+
_product_embeddings = clip_model.encode_text(open_clip.tokenize(product_prompts).to(device))
|
| 277 |
+
_product_embeddings = _product_embeddings / _product_embeddings.norm(dim=-1, keepdim=True)
|
| 278 |
+
|
| 279 |
+
_brand_embeddings = clip_model.encode_text(open_clip.tokenize(brand_prompts).to(device))
|
| 280 |
+
_brand_embeddings = _brand_embeddings / _brand_embeddings.norm(dim=-1, keepdim=True)
|
| 281 |
+
|
| 282 |
+
_character_embeddings = clip_model.encode_text(open_clip.tokenize(character_prompts).to(device))
|
| 283 |
+
_character_embeddings = _character_embeddings / _character_embeddings.norm(dim=-1, keepdim=True)
|
| 284 |
+
|
| 285 |
+
_detail_embeddings = clip_model.encode_text(open_clip.tokenize(detail_prompts).to(device))
|
| 286 |
+
_detail_embeddings = _detail_embeddings / _detail_embeddings.norm(dim=-1, keepdim=True)
|
| 287 |
+
|
| 288 |
+
def enhanced_zero_shot_classification(image: Image.Image):
|
| 289 |
+
"""Enhanced zero-shot classification across multiple categories."""
|
| 290 |
with torch.inference_mode():
|
| 291 |
img_tensor = clip_preprocess(image).unsqueeze(0).to(device)
|
| 292 |
img_emb = clip_model.encode_image(img_tensor)
|
| 293 |
img_emb = img_emb / img_emb.norm(dim=-1, keepdim=True)
|
| 294 |
+
|
| 295 |
+
results = {}
|
| 296 |
+
|
| 297 |
+
# Product classification
|
| 298 |
+
product_logits = 100.0 * img_emb @ _product_embeddings.T
|
| 299 |
+
product_probs = torch.softmax(product_logits, dim=-1)
|
| 300 |
+
top_products = torch.topk(product_probs, 3, dim=-1)
|
| 301 |
+
results['products'] = [(COMPREHENSIVE_PRODUCT_LABELS[idx], float(prob))
|
| 302 |
+
for prob, idx in zip(top_products.values[0], top_products.indices[0])]
|
| 303 |
+
|
| 304 |
+
# Brand classification
|
| 305 |
+
brand_logits = 100.0 * img_emb @ _brand_embeddings.T
|
| 306 |
+
brand_probs = torch.softmax(brand_logits, dim=-1)
|
| 307 |
+
top_brands = torch.topk(brand_probs, 3, dim=-1)
|
| 308 |
+
results['brands'] = [(COMPREHENSIVE_BRAND_LABELS[idx], float(prob))
|
| 309 |
+
for prob, idx in zip(top_brands.values[0], top_brands.indices[0])]
|
| 310 |
+
|
| 311 |
+
# Character classification
|
| 312 |
+
character_logits = 100.0 * img_emb @ _character_embeddings.T
|
| 313 |
+
character_probs = torch.softmax(character_logits, dim=-1)
|
| 314 |
+
top_characters = torch.topk(character_probs, 3, dim=-1)
|
| 315 |
+
results['characters'] = [(CHARACTER_LABELS[idx], float(prob))
|
| 316 |
+
for prob, idx in zip(top_characters.values[0], top_characters.indices[0])]
|
| 317 |
+
|
| 318 |
+
# Detail classification
|
| 319 |
+
detail_logits = 100.0 * img_emb @ _detail_embeddings.T
|
| 320 |
+
detail_probs = torch.softmax(detail_logits, dim=-1)
|
| 321 |
+
top_details = torch.topk(detail_probs, 5, dim=-1)
|
| 322 |
+
results['details'] = [(DETAIL_LABELS[idx], float(prob))
|
| 323 |
+
for prob, idx in zip(top_details.values[0], top_details.indices[0])]
|
| 324 |
+
|
| 325 |
+
return results
|
| 326 |
+
|
| 327 |
+
# === Enhanced BLIP captioning ===
|
| 328 |
+
def get_enhanced_caption(image: Image.Image):
|
| 329 |
+
"""Generate multiple captions with different prompts and combine insights."""
|
| 330 |
+
prompts = [
|
| 331 |
+
None, # Unconditional
|
| 332 |
+
"Describe this product in detail including its brand, type, colors, and any visible text or logos.",
|
| 333 |
+
"What are the key visual features and characteristics of this item?",
|
| 334 |
+
"Provide a comprehensive description of this object including materials, colors, and design elements."
|
| 335 |
+
]
|
| 336 |
+
|
| 337 |
+
captions = []
|
| 338 |
+
for prompt in prompts:
|
| 339 |
+
try:
|
| 340 |
+
if prompt is None:
|
| 341 |
+
inputs = blip_processor(image, return_tensors="pt").to(device)
|
| 342 |
+
else:
|
| 343 |
+
inputs = blip_processor(image, text=prompt, return_tensors="pt").to(device)
|
| 344 |
+
|
| 345 |
+
with torch.inference_mode():
|
| 346 |
+
out = blip_model.generate(
|
| 347 |
+
**inputs,
|
| 348 |
+
max_new_tokens=80,
|
| 349 |
+
num_beams=5,
|
| 350 |
+
do_sample=True,
|
| 351 |
+
temperature=0.7,
|
| 352 |
+
early_stopping=True,
|
| 353 |
+
no_repeat_ngram_size=3
|
| 354 |
+
)
|
| 355 |
+
|
| 356 |
+
caption = blip_processor.decode(out[0], skip_special_tokens=True).strip()
|
| 357 |
+
if prompt and prompt.lower() in caption.lower():
|
| 358 |
+
continue # Skip if caption just repeats the prompt
|
| 359 |
+
captions.append(caption)
|
| 360 |
+
|
| 361 |
+
except Exception as e:
|
| 362 |
+
print(f"Caption generation error: {e}")
|
| 363 |
+
continue
|
| 364 |
+
|
| 365 |
+
# Return the longest, most detailed caption
|
| 366 |
+
return max(captions, key=len) if captions else "Unable to generate caption"
|
| 367 |
+
|
| 368 |
+
# === Comprehensive analysis pipeline ===
|
| 369 |
+
def comprehensive_image_analysis(image: Image.Image):
|
| 370 |
+
"""Main analysis pipeline combining all techniques."""
|
| 371 |
+
# Resize for efficiency
|
| 372 |
+
if max(image.size) > 1024:
|
| 373 |
image = image.copy()
|
| 374 |
+
image.thumbnail((1024, 1024), Image.Resampling.LANCZOS)
|
| 375 |
+
|
| 376 |
+
analysis = {}
|
| 377 |
+
|
| 378 |
+
# Enhanced captioning
|
| 379 |
+
analysis['caption'] = get_enhanced_caption(image)
|
| 380 |
+
|
| 381 |
+
# VQA analysis
|
| 382 |
+
analysis['vqa'] = analyze_with_vqa(image)
|
| 383 |
+
|
| 384 |
+
# Zero-shot classification
|
| 385 |
+
analysis['classification'] = enhanced_zero_shot_classification(image)
|
| 386 |
+
|
| 387 |
+
# OCR analysis
|
| 388 |
+
analysis['ocr_text'] = extract_comprehensive_ocr(image)
|
| 389 |
+
|
| 390 |
+
# Color analysis
|
| 391 |
+
analysis['colors'] = extract_dominant_colors(image)
|
| 392 |
+
|
| 393 |
+
return analysis
|
| 394 |
+
|
| 395 |
+
# === Alt-text generation ===
|
| 396 |
+
def build_comprehensive_alt_text(analysis):
|
| 397 |
+
"""Build detailed alt-text from comprehensive analysis."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 398 |
parts = []
|
| 399 |
+
|
| 400 |
+
# Main caption
|
| 401 |
+
if analysis.get('caption'):
|
| 402 |
+
parts.append(f"Description: {analysis['caption']}")
|
| 403 |
+
|
| 404 |
+
# Product type (from classification)
|
| 405 |
+
products = analysis.get('classification', {}).get('products', [])
|
| 406 |
+
if products and products[0][1] > 0.3: # Confidence threshold
|
| 407 |
+
parts.append(f"Product type: {products[0][0]}")
|
| 408 |
+
|
| 409 |
+
# Brand (from classification and VQA)
|
| 410 |
+
brands = analysis.get('classification', {}).get('brands', [])
|
| 411 |
+
brand_vqa = analysis.get('vqa', {}).get('brand is visible in this image', '')
|
| 412 |
+
|
| 413 |
+
brand_text = ""
|
| 414 |
+
if brands and brands[0][1] > 0.25 and brands[0][0] not in ['unknown', 'generic', 'no brand visible']:
|
| 415 |
+
brand_text = brands[0][0]
|
| 416 |
+
elif brand_vqa and len(brand_vqa) > 2 and brand_vqa.lower() not in ['no', 'none', 'unknown']:
|
| 417 |
+
brand_text = brand_vqa
|
| 418 |
+
|
| 419 |
+
if brand_text:
|
| 420 |
+
parts.append(f"Brand: {brand_text}")
|
| 421 |
+
|
| 422 |
+
# Characters/People
|
| 423 |
+
characters = analysis.get('classification', {}).get('characters', [])
|
| 424 |
+
if characters and characters[0][1] > 0.4:
|
| 425 |
+
parts.append(f"Features: {characters[0][0]}")
|
| 426 |
+
|
| 427 |
+
# Colors
|
| 428 |
+
colors = analysis.get('colors', [])
|
| 429 |
+
if colors:
|
| 430 |
+
color_text = ", ".join(colors[:3]) # Top 3 colors
|
| 431 |
+
parts.append(f"Dominant colors: {color_text}")
|
| 432 |
+
|
| 433 |
+
# OCR text
|
| 434 |
+
ocr_text = analysis.get('ocr_text', '')
|
| 435 |
if ocr_text:
|
| 436 |
+
parts.append(f"Visible text: \"{ocr_text}\"")
|
| 437 |
+
|
| 438 |
+
# Key details
|
| 439 |
+
details = analysis.get('classification', {}).get('details', [])
|
| 440 |
+
high_conf_details = [detail for detail, conf in details if conf > 0.4]
|
| 441 |
+
if high_conf_details:
|
| 442 |
+
parts.append(f"Visual elements: {', '.join(high_conf_details[:3])}")
|
| 443 |
+
|
| 444 |
+
# VQA insights
|
| 445 |
+
vqa = analysis.get('vqa', {})
|
| 446 |
+
material = vqa.get('material does this product appear to be made of', '')
|
| 447 |
+
if material and len(material) > 2:
|
| 448 |
+
parts.append(f"Material: {material}")
|
| 449 |
+
|
| 450 |
+
background = vqa.get('background of this image', '')
|
| 451 |
+
if background and len(background) > 5 and background.lower() not in ['white', 'plain', 'simple']:
|
| 452 |
+
parts.append(f"Background: {background}")
|
| 453 |
+
|
| 454 |
+
# Combine all parts
|
| 455 |
+
alt_text = "; ".join(parts)
|
| 456 |
+
|
| 457 |
+
# Ensure proper ending
|
| 458 |
+
if not alt_text.endswith('.'):
|
| 459 |
+
alt_text += '.'
|
| 460 |
+
|
| 461 |
+
return alt_text
|
| 462 |
+
|
| 463 |
+
# === Initialize embeddings ===
|
| 464 |
+
print("Precomputing embeddings...")
|
| 465 |
+
precompute_embeddings()
|
| 466 |
+
print("Embeddings ready!")
|
| 467 |
+
|
| 468 |
+
# === Processing functions ===
|
|
|
|
|
|
|
| 469 |
def handle_single_image(file):
|
| 470 |
+
"""Process a single image file."""
|
| 471 |
try:
|
| 472 |
image = Image.open(file.name).convert("RGB")
|
| 473 |
except Exception as e:
|
| 474 |
return f"Failed to open image: {e}"
|
| 475 |
+
|
| 476 |
+
try:
|
| 477 |
+
analysis = comprehensive_image_analysis(image)
|
| 478 |
+
alt_text = build_comprehensive_alt_text(analysis)
|
| 479 |
+
return alt_text
|
| 480 |
+
except Exception as e:
|
| 481 |
+
return f"Analysis failed: {e}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 482 |
|
| 483 |
def preview_images_grid(files):
|
| 484 |
+
"""Generate preview grid for uploaded images."""
|
| 485 |
if not files:
|
| 486 |
return []
|
| 487 |
previews = []
|
| 488 |
+
for file in files[:12]: # Limit preview
|
| 489 |
try:
|
| 490 |
img = Image.open(file.name).convert("RGB")
|
| 491 |
+
img.thumbnail((200, 200), Image.Resampling.LANCZOS)
|
| 492 |
previews.append(img)
|
| 493 |
except:
|
| 494 |
continue
|
| 495 |
return previews
|
| 496 |
|
| 497 |
+
def process_images(files):
|
| 498 |
+
"""Process multiple images with controlled concurrency."""
|
| 499 |
if not files:
|
| 500 |
return "No images uploaded."
|
| 501 |
+
|
| 502 |
results = []
|
| 503 |
+
# Use limited concurrency to avoid GPU memory issues
|
| 504 |
+
with ThreadPoolExecutor(max_workers=1) as executor: # Sequential processing for stability
|
| 505 |
+
futures = {executor.submit(handle_single_image, f): idx for idx, f in enumerate(files[:8])}
|
| 506 |
+
|
| 507 |
for fut in as_completed(futures):
|
| 508 |
idx = futures[fut]
|
| 509 |
try:
|
| 510 |
+
alt_text = fut.result()
|
| 511 |
+
results.append(f"**Image {idx+1}:**\n{alt_text}\n")
|
| 512 |
except Exception as e:
|
| 513 |
+
results.append(f"**Image {idx+1}:** Failed to process - {e}\n")
|
| 514 |
+
|
| 515 |
+
# Sort by original order
|
| 516 |
+
results_sorted = sorted(results, key=lambda s: int(re.search(r"Image (\d+):", s).group(1)))
|
| 517 |
+
return "\n".join(results_sorted)
|
| 518 |
|
| 519 |
+
# === Gradio Interface ===
|
| 520 |
with gr.Blocks(
|
| 521 |
+
title="π Advanced Product Alt-Text Generator",
|
| 522 |
theme=gr.themes.Soft(),
|
| 523 |
css="""
|
| 524 |
+
.gradio-container { max-width: 1400px !important; }
|
| 525 |
+
.description-output {
|
| 526 |
+
font-family: 'Georgia', serif;
|
| 527 |
+
line-height: 1.6;
|
| 528 |
+
font-size: 14px;
|
| 529 |
+
}
|
| 530 |
+
.upload-section { border: 2px dashed #ccc; border-radius: 10px; padding: 20px; }
|
| 531 |
"""
|
| 532 |
) as demo:
|
| 533 |
+
|
| 534 |
+
gr.Markdown("""
|
| 535 |
+
# π Advanced Product Alt-Text Generator
|
| 536 |
+
|
| 537 |
+
**Enhanced with comprehensive analysis capabilities:**
|
| 538 |
+
- β
**Multi-model approach**: BLIP captioning + VQA + CLIP classification
|
| 539 |
+
- β
**Extensive object recognition**: 100+ product types, 50+ brands
|
| 540 |
+
- β
**Character & person detection**: Identifies people, mascots, characters
|
| 541 |
+
- β
**Advanced OCR**: Multi-language text extraction with preprocessing
|
| 542 |
+
- β
**Color analysis**: Dominant color extraction using K-means clustering
|
| 543 |
+
- β
**Visual question answering**: Asks specific questions about image content
|
| 544 |
+
- β
**Material & texture recognition**: Identifies materials and surface properties
|
| 545 |
+
|
| 546 |
+
Upload product images to generate detailed, accurate alt-text descriptions.
|
| 547 |
+
""")
|
| 548 |
+
|
| 549 |
with gr.Row():
|
| 550 |
+
with gr.Column(scale=1, elem_classes=["upload-section"]):
|
| 551 |
file_input = gr.File(
|
| 552 |
+
label="π€ Upload Product Images",
|
| 553 |
file_count="multiple",
|
| 554 |
file_types=["image"],
|
| 555 |
+
height=200,
|
| 556 |
)
|
| 557 |
+
generate_btn = gr.Button(
|
| 558 |
+
"π Generate Detailed Alt-Text",
|
| 559 |
+
variant="primary",
|
| 560 |
+
size="lg"
|
| 561 |
+
)
|
| 562 |
+
|
| 563 |
+
gr.Markdown("""
|
| 564 |
+
**Supported formats:** JPG, PNG, WebP, BMP
|
| 565 |
+
**Max images:** 8 per batch
|
| 566 |
+
**Recommended:** High-quality, well-lit product photos
|
| 567 |
+
""")
|
| 568 |
+
|
| 569 |
with gr.Column(scale=2):
|
| 570 |
gallery = gr.Gallery(
|
| 571 |
+
label="πΈ Image Preview",
|
| 572 |
columns=4,
|
| 573 |
rows=2,
|
| 574 |
+
height=350,
|
| 575 |
object_fit="contain"
|
| 576 |
)
|
| 577 |
+
|
| 578 |
output = gr.Textbox(
|
| 579 |
+
label="π Generated Alt-Text Descriptions",
|
| 580 |
+
lines=20,
|
| 581 |
show_copy_button=True,
|
| 582 |
elem_classes=["description-output"],
|
| 583 |
+
placeholder="Detailed alt-text descriptions will appear here after processing...",
|
| 584 |
+
max_lines=30
|
| 585 |
+
)
|
| 586 |
+
|
| 587 |
+
# Event handlers
|
| 588 |
+
file_input.change(
|
| 589 |
+
fn=preview_images_grid,
|
| 590 |
+
inputs=file_input,
|
| 591 |
+
outputs=gallery,
|
| 592 |
+
show_progress="hidden"
|
| 593 |
)
|
| 594 |
+
|
| 595 |
+
generate_btn.click(
|
| 596 |
+
fn=process_images,
|
| 597 |
+
inputs=file_input,
|
| 598 |
+
outputs=output,
|
| 599 |
+
show_progress=True
|
|
|
|
|
|
|
|
|
|
| 600 |
)
|
| 601 |
+
|
| 602 |
+
gr.Markdown("""
|
| 603 |
+
---
|
| 604 |
+
**π§ Technical Features:**
|
| 605 |
+
- **Enhanced BLIP**: Uses large model with multiple prompting strategies
|
| 606 |
+
- **Comprehensive CLIP**: 200+ labels across products, brands, characters, details
|
| 607 |
+
- **Smart OCR**: Multi-preprocessing approaches for better text recognition
|
| 608 |
+
- **Color Intelligence**: Scientific color naming with clustering analysis
|
| 609 |
+
- **VQA Integration**: Asks targeted questions for specific details
|
| 610 |
+
- **Robust Processing**: Error handling and fallbacks for reliable results
|
| 611 |
+
|
| 612 |
+
**π‘ Best Results:** Use clear, well-lit images with visible branding and text.
|
| 613 |
+
""")
|
| 614 |
|
| 615 |
if __name__ == "__main__":
|
| 616 |
+
demo.launch(
|
| 617 |
+
server_name="0.0.0.0",
|
| 618 |
+
server_port=7860,
|
| 619 |
+
share=False,
|
| 620 |
+
show_error=True
|
| 621 |
+
)
|