ALT / app.py
Workpam's picture
Update app.py
9aa248c verified
Raw History Blame Contribute Delete
9.12 kB
import gradio as gr
from PIL import Image
import torch
import hashlib
import json
from functools import lru_cache
# ---- Try loading LLaVA first (8-bit, low memory) ----
use_llava = False
use_blip2 = False
llava_model = None
llava_processor = None
blip2_model = None
blip2_processor = None
def safe_hash_image(img: Image.Image) -> str:
h = hashlib.sha256()
with img.convert("RGB") as im:
im_bytes = im.resize((256, 256)).tobytes()
h.update(im_bytes)
return h.hexdigest()
def try_load_llava():
global llava_model, llava_processor, use_llava
try:
from transformers import AutoProcessor, LlavaForConditionalGeneration
model_name = "llava-hf/llava-v1.6-mistral-7b-hf"
llava_processor = AutoProcessor.from_pretrained(model_name)
llava_model = LlavaForConditionalGeneration.from_pretrained(
model_name,
load_in_8bit=True, # quantize to 8-bit
device_map="auto", # let HF distribute
low_cpu_mem_usage=True,
trust_remote_code=True,
)
llava_model.eval()
use_llava = True
print("Loaded LLaVA in 8-bit mode.")
except Exception as e:
print(f"LLaVA load failed, will fallback to BLIP-2. Reason: {e}")
def try_load_blip2():
global blip2_model, blip2_processor, use_blip2
try:
from transformers import Blip2Processor, Blip2ForConditionalGeneration
fallback_name = "Salesforce/blip2-flan-t5-base"
blip2_processor = Blip2Processor.from_pretrained(fallback_name)
blip2_model = Blip2ForConditionalGeneration.from_pretrained(
fallback_name,
device_map="auto" if torch.cuda.is_available() else None,
torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
)
blip2_model.eval()
use_blip2 = True
print("Loaded BLIP-2 fallback model.")
except Exception as e:
print(f"BLIP-2 load failed too: {e}")
# Load models at startup
try_load_llava()
if not use_llava:
try_load_blip2()
elif not torch.cuda.is_available():
# If LLaVA is loaded but only CPU is present, might still be heavy; optionally fall back
# Keep LLaVA but warning in output if memory becomes an issue.
pass
# Common system instruction for structured output
STRUCTURE_INSTRUCTION = (
"Produce a JSON object with keys:\n"
"1. main_objects: list of primary objects.\n"
"2. visible_text: any readable text in quotes or null.\n"
"3. background: description of background excluding main objects.\n"
"4. colors_materials: dominant colors/textures/materials.\n"
"5. spatial_relationships: relative positions of objects.\n"
"6. uncertainties: things not certain.\n"
"7. summary_alt_text: concise one-sentence alt text.\n"
"After the JSON, add a short human-readable paragraph summarizing the image."
)
def generate_with_llava(image: Image.Image) -> str:
from transformers import AutoProcessor
# Resize moderately to save memory
max_side = 1024
if max(image.size) > max_side:
scale = max_side / max(image.size)
image = image.resize((int(image.width * scale), int(image.height * scale)), Image.BICUBIC)
conversation = [
{"role": "system", "content": STRUCTURE_INSTRUCTION},
{
"role": "user",
"content": [
{"type": "image", "image": image},
{
"type": "text",
"text": "Analyze and output as described."
},
],
},
]
with torch.inference_mode():
inputs = llava_processor.apply_chat_template(
conversation,
add_generation_prompt=True,
tokenize=True,
return_tensors="pt",
)
for k, v in inputs.items():
inputs[k] = v.to(llava_model.device)
gen_kwargs = dict(
**inputs,
max_new_tokens=150,
do_sample=True,
temperature=0.7,
top_p=0.9,
repetition_penalty=1.1,
early_stopping=True,
)
output_ids = llava_model.generate(**gen_kwargs)
raw = llava_processor.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
return raw
def generate_with_blip2(image: Image.Image) -> str:
from transformers import Blip2Processor
prompt = (
STRUCTURE_INSTRUCTION
+ " Analyze the image and respond."
)
# Resize a bit to keep CPU reasonable
max_side = 800 if not torch.cuda.is_available() else 1024
if max(image.size) > max_side:
scale = max_side / max(image.size)
image = image.resize((int(image.width * scale), int(image.height * scale)), Image.BICUBIC)
inputs = blip2_processor(images=image, text=prompt, return_tensors="pt")
for k, v in inputs.items():
inputs[k] = v.to(blip2_model.device) if hasattr(blip2_model, "device") else v
with torch.inference_mode():
generated_ids = blip2_model.generate(**inputs, max_new_tokens=140, early_stopping=True)
raw = blip2_processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip()
return raw
def extract_structured_and_narrative(raw_output: str) -> (dict, str):
structured = {}
narrative = ""
try:
first_brace = raw_output.find("{")
if first_brace != -1:
depth = 0
for idx in range(first_brace, len(raw_output)):
if raw_output[idx] == "{":
depth += 1
elif raw_output[idx] == "}":
depth -= 1
if depth == 0:
json_text = raw_output[first_brace: idx + 1]
structured = json.loads(json_text.replace("\n", " ").strip())
narrative = raw_output[idx + 1 :].strip()
break
if not structured:
structured = json.loads(raw_output)
else:
narrative = raw_output
except Exception:
structured = {
"main_objects": None,
"visible_text": None,
"background": None,
"colors_materials": None,
"spatial_relationships": None,
"uncertainties": None,
"summary_alt_text": None,
}
narrative = raw_output
if not structured.get("summary_alt_text"):
if narrative:
structured["summary_alt_text"] = narrative.split(".")[0].strip() + "."
else:
structured["summary_alt_text"] = "Description unavailable."
return structured, narrative
# Cache results per image hash to avoid recompute in session
@lru_cache(maxsize=128)
def generate_cached(image_hash: str, use_llava_flag: bool, use_blip2_flag: bool, raw_bytes: bytes) -> dict:
# raw_bytes is a dummy to make hashable; we only use image_hash to key.
# In practice, caller will pass image_bytes for uniqueness.
dummy_image = Image.frombytes("RGB", (1,1), b"\x00") # unused
# This wrapper won't be used; actual generation happens in wrapper below.
return {}
def pipeline(image: Image.Image) -> str:
if image is None:
return "No image provided."
# Decide which model to use
if use_llava:
try:
raw = generate_with_llava(image)
structured, narrative = extract_structured_and_narrative(raw)
note = "LLaVA (8-bit) used."
except Exception as e:
raw = f"LLaVA generation failed: {e}"
if use_blip2:
raw = generate_with_blip2(image)
structured, narrative = extract_structured_and_narrative(raw)
note = "Fell back to BLIP-2 after LLaVA error."
else:
return f"Both models failed. Last error: {e}"
elif use_blip2:
try:
raw = generate_with_blip2(image)
structured, narrative = extract_structured_and_narrative(raw)
note = "BLIP-2 used."
except Exception as e:
return f"BLIP-2 generation failed: {e}"
else:
return "No model loaded."
# Compose display
out = {
"structured": structured,
"narrative": narrative,
"note": note,
}
pretty = "Structured JSON:\n" + json.dumps(structured, indent=2, ensure_ascii=False)
pretty += f"\n\nNarrative summary:\n{narrative}\n\n(Note: {note})"
return pretty
# Gradio UI
with gr.Blocks() as iface:
gr.Markdown("## Structured Alt Text Generator — LLaVA with BLIP-2 Fallback")
with gr.Row():
inp = gr.Image(type="pil", label="Upload Image")
out = gr.Textbox(label="Output (JSON + Summary)", lines=18)
btn = gr.Button("Describe Image")
btn.click(fn=pipeline, inputs=inp, outputs=out)
gr.Markdown(
"👆 If LLaVA fails due to memory constraints, the system automatically falls back to BLIP-2. "
"For best performance on LLaVA, deploy with a GPU-backed Space."
)
if __name__ == "__main__":
iface.launch()