import gradio as gr from PIL import Image import torch import hashlib import json from functools import lru_cache # ---- Try loading LLaVA first (8-bit, low memory) ---- use_llava = False use_blip2 = False llava_model = None llava_processor = None blip2_model = None blip2_processor = None def safe_hash_image(img: Image.Image) -> str: h = hashlib.sha256() with img.convert("RGB") as im: im_bytes = im.resize((256, 256)).tobytes() h.update(im_bytes) return h.hexdigest() def try_load_llava(): global llava_model, llava_processor, use_llava try: from transformers import AutoProcessor, LlavaForConditionalGeneration model_name = "llava-hf/llava-v1.6-mistral-7b-hf" llava_processor = AutoProcessor.from_pretrained(model_name) llava_model = LlavaForConditionalGeneration.from_pretrained( model_name, load_in_8bit=True, # quantize to 8-bit device_map="auto", # let HF distribute low_cpu_mem_usage=True, trust_remote_code=True, ) llava_model.eval() use_llava = True print("Loaded LLaVA in 8-bit mode.") except Exception as e: print(f"LLaVA load failed, will fallback to BLIP-2. Reason: {e}") def try_load_blip2(): global blip2_model, blip2_processor, use_blip2 try: from transformers import Blip2Processor, Blip2ForConditionalGeneration fallback_name = "Salesforce/blip2-flan-t5-base" blip2_processor = Blip2Processor.from_pretrained(fallback_name) blip2_model = Blip2ForConditionalGeneration.from_pretrained( fallback_name, device_map="auto" if torch.cuda.is_available() else None, torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32, ) blip2_model.eval() use_blip2 = True print("Loaded BLIP-2 fallback model.") except Exception as e: print(f"BLIP-2 load failed too: {e}") # Load models at startup try_load_llava() if not use_llava: try_load_blip2() elif not torch.cuda.is_available(): # If LLaVA is loaded but only CPU is present, might still be heavy; optionally fall back # Keep LLaVA but warning in output if memory becomes an issue. pass # Common system instruction for structured output STRUCTURE_INSTRUCTION = ( "Produce a JSON object with keys:\n" "1. main_objects: list of primary objects.\n" "2. visible_text: any readable text in quotes or null.\n" "3. background: description of background excluding main objects.\n" "4. colors_materials: dominant colors/textures/materials.\n" "5. spatial_relationships: relative positions of objects.\n" "6. uncertainties: things not certain.\n" "7. summary_alt_text: concise one-sentence alt text.\n" "After the JSON, add a short human-readable paragraph summarizing the image." ) def generate_with_llava(image: Image.Image) -> str: from transformers import AutoProcessor # Resize moderately to save memory max_side = 1024 if max(image.size) > max_side: scale = max_side / max(image.size) image = image.resize((int(image.width * scale), int(image.height * scale)), Image.BICUBIC) conversation = [ {"role": "system", "content": STRUCTURE_INSTRUCTION}, { "role": "user", "content": [ {"type": "image", "image": image}, { "type": "text", "text": "Analyze and output as described." }, ], }, ] with torch.inference_mode(): inputs = llava_processor.apply_chat_template( conversation, add_generation_prompt=True, tokenize=True, return_tensors="pt", ) for k, v in inputs.items(): inputs[k] = v.to(llava_model.device) gen_kwargs = dict( **inputs, max_new_tokens=150, do_sample=True, temperature=0.7, top_p=0.9, repetition_penalty=1.1, early_stopping=True, ) output_ids = llava_model.generate(**gen_kwargs) raw = llava_processor.batch_decode(output_ids, skip_special_tokens=True)[0].strip() return raw def generate_with_blip2(image: Image.Image) -> str: from transformers import Blip2Processor prompt = ( STRUCTURE_INSTRUCTION + " Analyze the image and respond." ) # Resize a bit to keep CPU reasonable max_side = 800 if not torch.cuda.is_available() else 1024 if max(image.size) > max_side: scale = max_side / max(image.size) image = image.resize((int(image.width * scale), int(image.height * scale)), Image.BICUBIC) inputs = blip2_processor(images=image, text=prompt, return_tensors="pt") for k, v in inputs.items(): inputs[k] = v.to(blip2_model.device) if hasattr(blip2_model, "device") else v with torch.inference_mode(): generated_ids = blip2_model.generate(**inputs, max_new_tokens=140, early_stopping=True) raw = blip2_processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip() return raw def extract_structured_and_narrative(raw_output: str) -> (dict, str): structured = {} narrative = "" try: first_brace = raw_output.find("{") if first_brace != -1: depth = 0 for idx in range(first_brace, len(raw_output)): if raw_output[idx] == "{": depth += 1 elif raw_output[idx] == "}": depth -= 1 if depth == 0: json_text = raw_output[first_brace: idx + 1] structured = json.loads(json_text.replace("\n", " ").strip()) narrative = raw_output[idx + 1 :].strip() break if not structured: structured = json.loads(raw_output) else: narrative = raw_output except Exception: structured = { "main_objects": None, "visible_text": None, "background": None, "colors_materials": None, "spatial_relationships": None, "uncertainties": None, "summary_alt_text": None, } narrative = raw_output if not structured.get("summary_alt_text"): if narrative: structured["summary_alt_text"] = narrative.split(".")[0].strip() + "." else: structured["summary_alt_text"] = "Description unavailable." return structured, narrative # Cache results per image hash to avoid recompute in session @lru_cache(maxsize=128) def generate_cached(image_hash: str, use_llava_flag: bool, use_blip2_flag: bool, raw_bytes: bytes) -> dict: # raw_bytes is a dummy to make hashable; we only use image_hash to key. # In practice, caller will pass image_bytes for uniqueness. dummy_image = Image.frombytes("RGB", (1,1), b"\x00") # unused # This wrapper won't be used; actual generation happens in wrapper below. return {} def pipeline(image: Image.Image) -> str: if image is None: return "No image provided." # Decide which model to use if use_llava: try: raw = generate_with_llava(image) structured, narrative = extract_structured_and_narrative(raw) note = "LLaVA (8-bit) used." except Exception as e: raw = f"LLaVA generation failed: {e}" if use_blip2: raw = generate_with_blip2(image) structured, narrative = extract_structured_and_narrative(raw) note = "Fell back to BLIP-2 after LLaVA error." else: return f"Both models failed. Last error: {e}" elif use_blip2: try: raw = generate_with_blip2(image) structured, narrative = extract_structured_and_narrative(raw) note = "BLIP-2 used." except Exception as e: return f"BLIP-2 generation failed: {e}" else: return "No model loaded." # Compose display out = { "structured": structured, "narrative": narrative, "note": note, } pretty = "Structured JSON:\n" + json.dumps(structured, indent=2, ensure_ascii=False) pretty += f"\n\nNarrative summary:\n{narrative}\n\n(Note: {note})" return pretty # Gradio UI with gr.Blocks() as iface: gr.Markdown("## Structured Alt Text Generator — LLaVA with BLIP-2 Fallback") with gr.Row(): inp = gr.Image(type="pil", label="Upload Image") out = gr.Textbox(label="Output (JSON + Summary)", lines=18) btn = gr.Button("Describe Image") btn.click(fn=pipeline, inputs=inp, outputs=out) gr.Markdown( "👆 If LLaVA fails due to memory constraints, the system automatically falls back to BLIP-2. " "For best performance on LLaVA, deploy with a GPU-backed Space." ) if __name__ == "__main__": iface.launch()