Download app.py from Workpam/Alt_Text: direct link, hf CLI and curl.
- Browser
- Download file 16.4 kB
-
https://huggingface.co/spaces/Workpam/Alt_Text/resolve/main/app.py
- Command line
-
hf download hf://spaces/Workpam/Alt_Text/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Workpam/Alt_Text/resolve/main/app.py
16.4 kB
| import torch | |
| from PIL import Image | |
| import gradio as gr | |
| import easyocr | |
| import open_clip | |
| import cv2 | |
| import numpy as np | |
| import re | |
| from concurrent.futures import ThreadPoolExecutor, as_completed | |
| from sklearn.cluster import KMeans | |
| from transformers import ( | |
| BlipProcessor, | |
| BlipForConditionalGeneration, | |
| BlipForQuestionAnswering | |
| ) | |
| # === Device setup === | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| torch.backends.cudnn.benchmark = True | |
| print(f"Using device: {device}") | |
| # === Model initialization === | |
| # BLIP for captions | |
| blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large") | |
| blip_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large") | |
| blip_model.to(device).eval() | |
| # BLIP for VQA (optional) | |
| try: | |
| blip_vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") | |
| blip_vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base") | |
| blip_vqa_model.to(device).eval() | |
| VQA_AVAILABLE = True | |
| except Exception as e: | |
| print(f"VQA model not available: {e}") | |
| VQA_AVAILABLE = False | |
| if device.type == "cuda": | |
| blip_model = blip_model.half() | |
| if VQA_AVAILABLE: | |
| blip_vqa_model = blip_vqa_model.half() | |
| # CLIP for zero-shot classification | |
| clip_model, _, clip_preprocess = open_clip.create_model_and_transforms("ViT-L-14", pretrained="openai") | |
| clip_model.to(device).eval() | |
| if device.type == "cuda": | |
| clip_model = clip_model.half() | |
| # OCR reader | |
| ocr_reader = easyocr.Reader(["en", "es", "fr", "de", "it"], gpu=(device.type == "cuda")) | |
| # === Label sets (unchanged) === | |
| COMPREHENSIVE_PRODUCT_LABELS = [ | |
| "trucker hat", "baseball cap", "beanie", "cap", "hat", "sunglasses", "watch", "jewelry", | |
| "necklace", "bracelet", "ring", "earrings", "bag", "backpack", "purse", "wallet", | |
| "belt", "scarf", "gloves", "shoes", "sneakers", "boots", "sandals", "heels", | |
| "smartphone", "laptop", "tablet", "headphones", "earbuds", "speaker", "camera", | |
| "gaming console", "controller", "keyboard", "mouse", "monitor", "TV", "remote", | |
| "charger", "cable", "smartwatch", "fitness tracker", "drone", "robot", | |
| "board game", "card game", "puzzle", "action figure", "doll", "toy car", "LEGO", | |
| "building blocks", "stuffed animal", "teddy bear", "toy robot", "game controller", | |
| "dice", "playing cards", "collectible card", "miniature figure", "model kit", | |
| "perfume bottle", "cologne", "lipstick", "mascara", "foundation", "powder compact", | |
| "nail polish", "skincare bottle", "lotion", "cream jar", "soap", "shampoo", | |
| "conditioner", "toothbrush", "razor", "makeup brush", "mirror", "hair dryer", | |
| "water bottle", "soda can", "beer bottle", "wine bottle", "coffee mug", "tea cup", | |
| "food package", "cereal box", "snack bag", "candy bar", "chocolate", "cookie package", | |
| "book", "magazine", "comic book", "DVD", "Blu-ray", "CD", "vinyl record", | |
| "notebook", "journal", "planner", "calendar", "poster", "artwork", "photo frame", | |
| "basketball", "football", "soccer ball", "tennis ball", "baseball", "golf ball", | |
| "dumbbells", "yoga mat", "water bottle", "sports equipment", "helmet", "gloves", | |
| "coffee maker", "blender", "toaster", "microwave", "pan", "pot", "plate", "bowl", | |
| "cup", "glass", "utensils", "knife", "cutting board", "storage container", | |
| "paint brush", "marker", "pen", "pencil", "crayon", "paint tube", "canvas", | |
| "sketchbook", "ruler", "scissors", "glue", "tape", "stickers", | |
| "box", "package", "container", "jar", "bottle", "can", "tube", "pouch", "bag" | |
| ] | |
| COMPREHENSIVE_BRAND_LABELS = [ | |
| "Apple", "Samsung", "Google", "Microsoft", "Sony", "Nintendo", "PlayStation", "Xbox", | |
| "HP", "Dell", "Lenovo", "ASUS", "Acer", "Canon", "Nikon", "GoPro", "DJI", | |
| "Nike", "Adidas", "Puma", "Under Armour", "Reebok", "Converse", "Vans", "New Balance", | |
| "Ray-Ban", "Oakley", "Rolex", "Casio", "Fossil", "Michael Kors", "Coach", "Louis Vuitton", | |
| "L'Oreal", "Maybelline", "MAC", "Sephora", "Clinique", "Estee Lauder", "Chanel", | |
| "Dior", "YSL", "Urban Decay", "Too Faced", "NARS", "Fenty Beauty", | |
| "Hasbro", "Mattel", "LEGO", "Ravensburger", "Fantasy Flight Games", "Asmodee", | |
| "Days of Wonder", "CMON", "Wizards of the Coast", "Pokémon", "Yu-Gi-Oh", | |
| "Coca-Cola", "Pepsi", "Starbucks", "Red Bull", "Monster", "Gatorade", "Powerade", | |
| "Nestle", "Hershey's", "Mars", "Ferrero", "Lindt", "Godiva", | |
| "Crayola", "Sharpie", "Prismacolor", "Winsor & Newton", "Molotow", "Posca", | |
| "Staedtler", "Faber-Castell", "Sakura", "Tombow", "Pilot", "Uni-ball", | |
| "unknown", "generic", "no brand visible" | |
| ] | |
| CHARACTER_LABELS = [ | |
| "person", "man", "woman", "child", "baby", "elderly person", "athlete", "professional", | |
| "cartoon character", "anime character", "superhero", "mascot", "celebrity", "model", | |
| "fantasy character", "sci-fi character", "historical figure", "fictional character" | |
| ] | |
| DETAIL_LABELS = [ | |
| "text", "logo", "brand name", "price tag", "barcode", "QR code", "warning label", | |
| "ingredients list", "nutritional information", "instructions", "serial number", | |
| "model number", "size information", "color variants", "pattern", "texture", | |
| "metallic finish", "matte finish", "glossy finish", "transparent", "translucent" | |
| ] | |
| # === Basic color palette for mapping === | |
| BASE_COLORS = { | |
| 'red': (255, 0, 0), | |
| 'green': (0, 128, 0), | |
| 'blue': (0, 0, 255), | |
| 'yellow': (255, 255, 0), | |
| 'orange': (255, 165, 0), | |
| 'purple': (128, 0, 128), | |
| 'pink': (255, 192, 203), | |
| 'brown': (165, 42, 42), | |
| 'gray': (128, 128, 128), | |
| 'black': (0, 0, 0), | |
| 'white': (255, 255, 255), | |
| } | |
| def get_basic_color_name(rgb_tuple): | |
| """Map any RGB to nearest basic color name.""" | |
| best, best_name = float('inf'), 'unknown' | |
| for name, ref in BASE_COLORS.items(): | |
| dist = sum((a - b) ** 2 for a, b in zip(rgb_tuple, ref)) ** 0.5 | |
| if dist < best: | |
| best, best_name = dist, name | |
| return best_name | |
| def extract_dominant_colors(image: Image.Image, n_colors=3): | |
| """Use k-means on downsampled pixels, then map to basic colors.""" | |
| arr = np.array(image.resize((150, 150))).reshape(-1, 3) | |
| # drop pure black/white | |
| mask = ~((arr == [255,255,255]).all(axis=1) | (arr == [0,0,0]).all(axis=1)) | |
| arr = arr[mask] if mask.sum() > 0 else arr | |
| if len(arr) < n_colors: | |
| return [] | |
| kmeans = KMeans(n_clusters=n_colors, random_state=42, n_init=10).fit(arr) | |
| colors = [] | |
| for center in kmeans.cluster_centers_: | |
| rgb = tuple(map(int, center)) | |
| colors.append(get_basic_color_name(rgb)) | |
| # dedupe and preserve order | |
| seen = set() | |
| result = [] | |
| for c in colors: | |
| if c not in seen: | |
| seen.add(c) | |
| result.append(c) | |
| return result | |
| # === Zero-shot classification setup === | |
| def precompute_embeddings(): | |
| global _prod_emb, _brand_emb, _char_emb, _detail_emb | |
| prompts_prod = [f"a photo of a {l}" for l in COMPREHENSIVE_PRODUCT_LABELS] | |
| prompts_brand = [f"a {b} product" for b in COMPREHENSIVE_BRAND_LABELS] | |
| prompts_char = [f"a photo of a {c}" for c in CHARACTER_LABELS] | |
| prompts_det = [f"an image showing {d}" for d in DETAIL_LABELS] | |
| with torch.inference_mode(): | |
| _prod_emb = clip_model.encode_text(open_clip.tokenize(prompts_prod).to(device)) | |
| _prod_emb /= _prod_emb.norm(dim=-1, keepdim=True) | |
| _brand_emb = clip_model.encode_text(open_clip.tokenize(prompts_brand).to(device)) | |
| _brand_emb /= _brand_emb.norm(dim=-1, keepdim=True) | |
| _char_emb = clip_model.encode_text(open_clip.tokenize(prompts_char).to(device)) | |
| _char_emb /= _char_emb.norm(dim=-1, keepdim=True) | |
| _det_emb = clip_model.encode_text(open_clip.tokenize(prompts_det).to(device)) | |
| _det_emb /= _det_emb.norm(dim=-1, keepdim=True) | |
| def enhanced_zero_shot_classification(image: Image.Image): | |
| with torch.inference_mode(): | |
| img_t = clip_preprocess(image).unsqueeze(0).to(device) | |
| img_e = clip_model.encode_image(img_t) | |
| img_e /= img_e.norm(dim=-1, keepdim=True) | |
| results = {} | |
| for labels, emb, key, topk in [ | |
| (COMPREHENSIVE_PRODUCT_LABELS, _prod_emb, 'products', 3), | |
| (COMPREHENSIVE_BRAND_LABELS, _brand_emb, 'brands', 3), | |
| (CHARACTER_LABELS, _char_emb, 'characters', 3), | |
| (DETAIL_LABELS, _det_emb, 'details', 5), | |
| ]: | |
| logits = (100.0 * img_e @ emb.T) | |
| probs = torch.softmax(logits, dim=-1) | |
| top = torch.topk(probs, topk, dim=-1) | |
| results[key] = [ | |
| (labels[idx], float(prob)) | |
| for prob, idx in zip(top.values[0], top.indices[0]) | |
| ] | |
| return results | |
| # === BLIP captioning with enhanced prompts === | |
| DETAILED_PROMPTS = [ | |
| "Provide a detailed, comprehensive description of this image including all visual elements, characters, objects, text, colors, setting, and artistic style.", | |
| "Describe this image in great detail, mentioning the title, characters' appearance, clothing, expressions, objects they're holding, background elements, and any text visible.", | |
| "Write a thorough description of this cover art or product image, including character details, clothing, facial expressions, objects, background setting, colors, and any visible text or branding.", | |
| "Analyze this image completely: describe the main subject, their appearance and clothing, any objects or accessories, the background setting, color scheme, artistic style, and all visible text." | |
| ] | |
| def get_enhanced_caption(image: Image.Image): | |
| captions = [] | |
| # unconditional baseline | |
| try: | |
| inputs = blip_processor(image, return_tensors="pt").to(device) | |
| out = blip_model.generate( | |
| **inputs, max_new_tokens=150, num_beams=8, | |
| do_sample=True, temperature=0.8, top_p=0.9, | |
| no_repeat_ngram_size=2, repetition_penalty=1.1 | |
| ) | |
| captions.append(blip_processor.decode(out[0], skip_special_tokens=True).strip()) | |
| except Exception as e: | |
| print(f"Baseline caption error: {e}") | |
| # detailed prompts | |
| for prompt in DETAILED_PROMPTS: | |
| try: | |
| inputs = blip_processor(image, text=prompt, return_tensors="pt").to(device) | |
| out = blip_model.generate( | |
| **inputs, max_new_tokens=200, num_beams=8, | |
| do_sample=True, temperature=0.7, top_p=0.85, | |
| no_repeat_ngram_size=3, repetition_penalty=1.15 | |
| ) | |
| captions.append(blip_processor.decode(out[0], skip_special_tokens=True).strip()) | |
| except Exception as e: | |
| print(f"Prompt caption error: {e}") | |
| if not captions: | |
| return "Unable to generate a caption." | |
| # pick the most content-rich (longest) caption | |
| best = max(captions, key=lambda s: len(s.split())) | |
| best = re.sub(r'^(a|an|the)\s+', '', best, flags=re.IGNORECASE).strip().capitalize() | |
| return best if best.endswith('.') else best + '.' | |
| # === OCR === | |
| def extract_comprehensive_ocr(image: Image.Image): | |
| try: | |
| arr = np.array(image) | |
| gray = cv2.cvtColor(arr, cv2.COLOR_RGB2GRAY) | |
| preproc = [arr] | |
| clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8)).apply(gray) | |
| preproc.append(cv2.cvtColor(clahe, cv2.COLOR_GRAY2RGB)) | |
| _, th = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) | |
| preproc.append(cv2.cvtColor(th, cv2.COLOR_GRAY2RGB)) | |
| texts = [] | |
| for p in preproc: | |
| for _, txt, conf in ocr_reader.readtext(p, detail=1, paragraph=True): | |
| if conf > 0.3 and len(txt.strip()) > 1: | |
| texts.append((txt.strip(), conf)) | |
| # dedupe | |
| seen = set() | |
| unique = [] | |
| for txt, conf in sorted(texts, key=lambda x: x[1], reverse=True): | |
| key = re.sub(r'\W+', '', txt.lower()) | |
| if key not in seen: | |
| seen.add(key) | |
| unique.append(txt) | |
| return " | ".join(unique[:5]) | |
| except Exception as e: | |
| print(f"OCR error: {e}") | |
| return "" | |
| # === Comprehensive pipeline === | |
| def comprehensive_image_analysis(image: Image.Image): | |
| # resize for efficiency | |
| if max(image.size) > 1024: | |
| image = image.copy() | |
| image.thumbnail((1024,1024), Image.Resampling.LANCZOS) | |
| return { | |
| 'caption': get_enhanced_caption(image), | |
| 'classification': enhanced_zero_shot_classification(image), | |
| 'ocr_text': extract_comprehensive_ocr(image), | |
| 'colors': extract_dominant_colors(image) | |
| } | |
| # === Build narrative alt-text === | |
| def build_narrative_alt_text(analysis): | |
| main_desc = analysis['caption'] | |
| parts = [] | |
| # OCR text | |
| ocr = analysis['ocr_text'] | |
| if ocr: | |
| parts.append(f'Visible text: "{ocr}"') | |
| # Dominant colors | |
| cols = analysis['colors'] | |
| if cols: | |
| if len(cols) == 1: | |
| parts.append(f"The primary color is {cols[0]}") | |
| else: | |
| col_str = ", ".join(cols[:-1]) + f", and {cols[-1]}" | |
| parts.append(f"The color palette features {col_str}") | |
| # Brand (only if high confidence) | |
| brands = analysis['classification']['brands'] | |
| if brands and brands[0][1] > 0.6: | |
| parts.append(f"Published by {brands[0][0]}") | |
| # Combine | |
| final = main_desc | |
| for p in parts: | |
| final += " " + p + "." | |
| final = re.sub(r'\s+', ' ', final).strip() | |
| if not final.endswith('.'): | |
| final += '.' | |
| return final | |
| # === Initialize embeddings === | |
| print("Precomputing embeddings...") | |
| precompute_embeddings() | |
| print("Embeddings ready!") | |
| # === Gradio interface === | |
| def handle_single_image(file): | |
| try: | |
| img = Image.open(file.name).convert("RGB") | |
| except Exception as e: | |
| return f"Failed to open image: {e}" | |
| try: | |
| analysis = comprehensive_image_analysis(img) | |
| return build_narrative_alt_text(analysis) | |
| except Exception as e: | |
| return f"Analysis failed: {e}" | |
| def preview_images_grid(files): | |
| previews = [] | |
| for f in (files or [])[:12]: | |
| try: | |
| img = Image.open(f.name).convert("RGB") | |
| img.thumbnail((200,200), Image.Resampling.LANCZOS) | |
| previews.append(img) | |
| except: | |
| continue | |
| return previews | |
| def process_images(files): | |
| if not files: | |
| return "No images uploaded." | |
| results = [] | |
| with ThreadPoolExecutor(max_workers=1) as exe: | |
| futures = {exe.submit(handle_single_image, f): i for i, f in enumerate(files[:8])} | |
| for fut in as_completed(futures): | |
| idx = futures[fut] | |
| try: | |
| results.append(f"**Image {idx+1}:**\n{fut.result()}\n") | |
| except Exception as e: | |
| results.append(f"**Image {idx+1}:** Failed - {e}\n") | |
| return "\n".join(sorted(results, key=lambda s: int(re.search(r"Image (\d+):", s).group(1)))) | |
| with gr.Blocks( | |
| title="🔍 Optimized Narrative Alt-Text Generator", | |
| theme=gr.themes.Soft(), | |
| css=""" | |
| .gradio-container { max-width:1400px!important; } | |
| .description-output { font-family:'Georgia',serif; line-height:1.6; font-size:14px; } | |
| .upload-section { border:2px dashed #ccc; border-radius:10px; padding:20px; } | |
| """ | |
| ) as demo: | |
| gr.Markdown(""" | |
| # 🖼️ Optimized Alt-Text Generator | |
| - Improved basic color mapping | |
| - Brand only if high confidence (≥60%) | |
| - Removed unreliable VQA-brand/material | |
| - K-means → basic colors for clearer palettes | |
| """) | |
| with gr.Row(): | |
| with gr.Column(scale=1, elem_classes=["upload-section"]): | |
| inp = gr.File(label="Upload Images", file_count="multiple", file_types=["image"], height=200) | |
| btn = gr.Button("Generate Alt-Text", variant="primary", size="lg") | |
| with gr.Column(scale=2): | |
| gal = gr.Gallery(label="Preview", columns=4, rows=2, height=350, object_fit="contain") | |
| out = gr.Textbox(label="Generated Alt-Text", lines=20, show_copy_button=True, | |
| elem_classes=["description-output"], placeholder="...") | |
| inp.change(preview_images_grid, inp, gal, show_progress="hidden") | |
| btn.click(process_images, inp, out, show_progress=True) | |
| if __name__ == "__main__": | |
| demo.launch(server_name="0.0.0.0", server_port=7860, share=False) | |