Download app.py from Workpam/ALT: direct link, hf CLI and curl.
- Browser
- Download file 9.12 kB
-
https://huggingface.co/spaces/Workpam/ALT/resolve/main/app.py
- Command line
-
hf download hf://spaces/Workpam/ALT/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Workpam/ALT/resolve/main/app.py
9.12 kB
| import gradio as gr | |
| from PIL import Image | |
| import torch | |
| import hashlib | |
| import json | |
| from functools import lru_cache | |
| # ---- Try loading LLaVA first (8-bit, low memory) ---- | |
| use_llava = False | |
| use_blip2 = False | |
| llava_model = None | |
| llava_processor = None | |
| blip2_model = None | |
| blip2_processor = None | |
| def safe_hash_image(img: Image.Image) -> str: | |
| h = hashlib.sha256() | |
| with img.convert("RGB") as im: | |
| im_bytes = im.resize((256, 256)).tobytes() | |
| h.update(im_bytes) | |
| return h.hexdigest() | |
| def try_load_llava(): | |
| global llava_model, llava_processor, use_llava | |
| try: | |
| from transformers import AutoProcessor, LlavaForConditionalGeneration | |
| model_name = "llava-hf/llava-v1.6-mistral-7b-hf" | |
| llava_processor = AutoProcessor.from_pretrained(model_name) | |
| llava_model = LlavaForConditionalGeneration.from_pretrained( | |
| model_name, | |
| load_in_8bit=True, # quantize to 8-bit | |
| device_map="auto", # let HF distribute | |
| low_cpu_mem_usage=True, | |
| trust_remote_code=True, | |
| ) | |
| llava_model.eval() | |
| use_llava = True | |
| print("Loaded LLaVA in 8-bit mode.") | |
| except Exception as e: | |
| print(f"LLaVA load failed, will fallback to BLIP-2. Reason: {e}") | |
| def try_load_blip2(): | |
| global blip2_model, blip2_processor, use_blip2 | |
| try: | |
| from transformers import Blip2Processor, Blip2ForConditionalGeneration | |
| fallback_name = "Salesforce/blip2-flan-t5-base" | |
| blip2_processor = Blip2Processor.from_pretrained(fallback_name) | |
| blip2_model = Blip2ForConditionalGeneration.from_pretrained( | |
| fallback_name, | |
| device_map="auto" if torch.cuda.is_available() else None, | |
| torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32, | |
| ) | |
| blip2_model.eval() | |
| use_blip2 = True | |
| print("Loaded BLIP-2 fallback model.") | |
| except Exception as e: | |
| print(f"BLIP-2 load failed too: {e}") | |
| # Load models at startup | |
| try_load_llava() | |
| if not use_llava: | |
| try_load_blip2() | |
| elif not torch.cuda.is_available(): | |
| # If LLaVA is loaded but only CPU is present, might still be heavy; optionally fall back | |
| # Keep LLaVA but warning in output if memory becomes an issue. | |
| pass | |
| # Common system instruction for structured output | |
| STRUCTURE_INSTRUCTION = ( | |
| "Produce a JSON object with keys:\n" | |
| "1. main_objects: list of primary objects.\n" | |
| "2. visible_text: any readable text in quotes or null.\n" | |
| "3. background: description of background excluding main objects.\n" | |
| "4. colors_materials: dominant colors/textures/materials.\n" | |
| "5. spatial_relationships: relative positions of objects.\n" | |
| "6. uncertainties: things not certain.\n" | |
| "7. summary_alt_text: concise one-sentence alt text.\n" | |
| "After the JSON, add a short human-readable paragraph summarizing the image." | |
| ) | |
| def generate_with_llava(image: Image.Image) -> str: | |
| from transformers import AutoProcessor | |
| # Resize moderately to save memory | |
| max_side = 1024 | |
| if max(image.size) > max_side: | |
| scale = max_side / max(image.size) | |
| image = image.resize((int(image.width * scale), int(image.height * scale)), Image.BICUBIC) | |
| conversation = [ | |
| {"role": "system", "content": STRUCTURE_INSTRUCTION}, | |
| { | |
| "role": "user", | |
| "content": [ | |
| {"type": "image", "image": image}, | |
| { | |
| "type": "text", | |
| "text": "Analyze and output as described." | |
| }, | |
| ], | |
| }, | |
| ] | |
| with torch.inference_mode(): | |
| inputs = llava_processor.apply_chat_template( | |
| conversation, | |
| add_generation_prompt=True, | |
| tokenize=True, | |
| return_tensors="pt", | |
| ) | |
| for k, v in inputs.items(): | |
| inputs[k] = v.to(llava_model.device) | |
| gen_kwargs = dict( | |
| **inputs, | |
| max_new_tokens=150, | |
| do_sample=True, | |
| temperature=0.7, | |
| top_p=0.9, | |
| repetition_penalty=1.1, | |
| early_stopping=True, | |
| ) | |
| output_ids = llava_model.generate(**gen_kwargs) | |
| raw = llava_processor.batch_decode(output_ids, skip_special_tokens=True)[0].strip() | |
| return raw | |
| def generate_with_blip2(image: Image.Image) -> str: | |
| from transformers import Blip2Processor | |
| prompt = ( | |
| STRUCTURE_INSTRUCTION | |
| + " Analyze the image and respond." | |
| ) | |
| # Resize a bit to keep CPU reasonable | |
| max_side = 800 if not torch.cuda.is_available() else 1024 | |
| if max(image.size) > max_side: | |
| scale = max_side / max(image.size) | |
| image = image.resize((int(image.width * scale), int(image.height * scale)), Image.BICUBIC) | |
| inputs = blip2_processor(images=image, text=prompt, return_tensors="pt") | |
| for k, v in inputs.items(): | |
| inputs[k] = v.to(blip2_model.device) if hasattr(blip2_model, "device") else v | |
| with torch.inference_mode(): | |
| generated_ids = blip2_model.generate(**inputs, max_new_tokens=140, early_stopping=True) | |
| raw = blip2_processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip() | |
| return raw | |
| def extract_structured_and_narrative(raw_output: str) -> (dict, str): | |
| structured = {} | |
| narrative = "" | |
| try: | |
| first_brace = raw_output.find("{") | |
| if first_brace != -1: | |
| depth = 0 | |
| for idx in range(first_brace, len(raw_output)): | |
| if raw_output[idx] == "{": | |
| depth += 1 | |
| elif raw_output[idx] == "}": | |
| depth -= 1 | |
| if depth == 0: | |
| json_text = raw_output[first_brace: idx + 1] | |
| structured = json.loads(json_text.replace("\n", " ").strip()) | |
| narrative = raw_output[idx + 1 :].strip() | |
| break | |
| if not structured: | |
| structured = json.loads(raw_output) | |
| else: | |
| narrative = raw_output | |
| except Exception: | |
| structured = { | |
| "main_objects": None, | |
| "visible_text": None, | |
| "background": None, | |
| "colors_materials": None, | |
| "spatial_relationships": None, | |
| "uncertainties": None, | |
| "summary_alt_text": None, | |
| } | |
| narrative = raw_output | |
| if not structured.get("summary_alt_text"): | |
| if narrative: | |
| structured["summary_alt_text"] = narrative.split(".")[0].strip() + "." | |
| else: | |
| structured["summary_alt_text"] = "Description unavailable." | |
| return structured, narrative | |
| # Cache results per image hash to avoid recompute in session | |
| def generate_cached(image_hash: str, use_llava_flag: bool, use_blip2_flag: bool, raw_bytes: bytes) -> dict: | |
| # raw_bytes is a dummy to make hashable; we only use image_hash to key. | |
| # In practice, caller will pass image_bytes for uniqueness. | |
| dummy_image = Image.frombytes("RGB", (1,1), b"\x00") # unused | |
| # This wrapper won't be used; actual generation happens in wrapper below. | |
| return {} | |
| def pipeline(image: Image.Image) -> str: | |
| if image is None: | |
| return "No image provided." | |
| # Decide which model to use | |
| if use_llava: | |
| try: | |
| raw = generate_with_llava(image) | |
| structured, narrative = extract_structured_and_narrative(raw) | |
| note = "LLaVA (8-bit) used." | |
| except Exception as e: | |
| raw = f"LLaVA generation failed: {e}" | |
| if use_blip2: | |
| raw = generate_with_blip2(image) | |
| structured, narrative = extract_structured_and_narrative(raw) | |
| note = "Fell back to BLIP-2 after LLaVA error." | |
| else: | |
| return f"Both models failed. Last error: {e}" | |
| elif use_blip2: | |
| try: | |
| raw = generate_with_blip2(image) | |
| structured, narrative = extract_structured_and_narrative(raw) | |
| note = "BLIP-2 used." | |
| except Exception as e: | |
| return f"BLIP-2 generation failed: {e}" | |
| else: | |
| return "No model loaded." | |
| # Compose display | |
| out = { | |
| "structured": structured, | |
| "narrative": narrative, | |
| "note": note, | |
| } | |
| pretty = "Structured JSON:\n" + json.dumps(structured, indent=2, ensure_ascii=False) | |
| pretty += f"\n\nNarrative summary:\n{narrative}\n\n(Note: {note})" | |
| return pretty | |
| # Gradio UI | |
| with gr.Blocks() as iface: | |
| gr.Markdown("## Structured Alt Text Generator — LLaVA with BLIP-2 Fallback") | |
| with gr.Row(): | |
| inp = gr.Image(type="pil", label="Upload Image") | |
| out = gr.Textbox(label="Output (JSON + Summary)", lines=18) | |
| btn = gr.Button("Describe Image") | |
| btn.click(fn=pipeline, inputs=inp, outputs=out) | |
| gr.Markdown( | |
| "👆 If LLaVA fails due to memory constraints, the system automatically falls back to BLIP-2. " | |
| "For best performance on LLaVA, deploy with a GPU-backed Space." | |
| ) | |
| if __name__ == "__main__": | |
| iface.launch() | |