Workpam commited on
Commit
938b2c6
·
verified ·
1 Parent(s): 67849f9

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +52 -52
app.py CHANGED
@@ -1,69 +1,69 @@
1
  import gradio as gr
2
- from transformers import Blip2Processor, Blip2ForConditionalGeneration
3
- from PIL import Image
4
  import torch
 
 
5
 
6
- # Use GPU if available for faster inference
7
  device = "cuda" if torch.cuda.is_available() else "cpu"
 
8
 
9
- # Load BLIP-2 XL model for top-tier accuracy
10
- processor = Blip2Processor.from_pretrained("Salesforce/blip2-flan-t5-xl")
11
- model = (
12
- Blip2ForConditionalGeneration.from_pretrained(
13
- "Salesforce/blip2-flan-t5-xl",
14
- torch_dtype=torch.float16 if device == "cuda" else torch.float32
15
- )
16
- .to(device)
17
- )
18
 
19
  def generate_alt_text(files):
20
  """
21
- Generates descriptive alt-text captions for uploaded images using BLIP-2 XL.
22
  """
23
  captions = []
24
  for f in files:
25
- path = f.name if hasattr(f, "name") else f
26
  image = Image.open(path).convert("RGB")
27
- inputs = processor(images=image, return_tensors="pt").to(device)
 
 
 
 
 
 
28
  with torch.no_grad():
29
- outputs = model.generate(**inputs, max_new_tokens=50)
30
- text = processor.decode(outputs[0], skip_special_tokens=True)
31
- captions.append(text.strip())
32
  return "\n".join([f"{i+1}. {cap}" for i, cap in enumerate(captions)])
33
 
34
- # Build Gradio interface
35
- def main():
36
- with gr.Blocks(title="Efficient & Accurate Alt Text Generator") as demo:
37
- gr.Markdown("# Alt‑Text Generator (BLIP‑2 XL)")
38
- gr.Markdown(
39
- "Upload multiple images for fast, accurate descriptions powered by Salesforce BLIP‑2 XL."
40
- )
41
- file_input = gr.File(
42
- label="Upload Images",
43
- file_count="multiple",
44
- file_types=["image"]
45
- )
46
- output_box = gr.Textbox(
47
- label="Generated Alt Texts",
48
- lines=10,
49
- placeholder="Your generated descriptions will appear here...",
50
- elem_id="output_box"
51
- )
52
- with gr.Row():
53
- generate_btn = gr.Button("Generate Alt Text")
54
- copy_btn = gr.Button("Copy to Clipboard")
55
- generate_btn.click(
56
- fn=generate_alt_text,
57
- inputs=file_input,
58
- outputs=output_box
59
- )
60
- copy_btn.click(
61
- fn=None,
62
- inputs=None,
63
- outputs=None,
64
- js="() => { navigator.clipboard.writeText(document.getElementById('output_box').value); }"
65
- )
66
- demo.launch()
67
 
68
  if __name__ == "__main__":
69
- main()
 
1
  import gradio as gr
2
+ from transformers import InstructBlipProcessor, InstructBlipForConditionalGeneration
 
3
  import torch
4
+ from PIL import Image
5
+ import easyocr
6
 
7
+ # Device and OCR setup
8
  device = "cuda" if torch.cuda.is_available() else "cpu"
9
+ reader = easyocr.Reader(['en'], gpu=(device=="cuda"))
10
 
11
+ # Load InstructBLIP Vicuna-7b model for advanced multimodal instruction
12
+ model_name = "Salesforce/instructblip-vicuna-7b"
13
+ processor = InstructBlipProcessor.from_pretrained(model_name)
14
+ model = InstructBlipForConditionalGeneration.from_pretrained(model_name).to(device)
 
 
 
 
 
15
 
16
  def generate_alt_text(files):
17
  """
18
+ Generates alt-text that accurately describes each image and includes any visible text.
19
  """
20
  captions = []
21
  for f in files:
22
+ path = f.name if hasattr(f, 'name') else f
23
  image = Image.open(path).convert("RGB")
24
+ # 1. OCR to extract on-image text
25
+ ocr_results = reader.readtext(path, detail=0)
26
+ visible_text = ' '.join(ocr_results).strip() or "<no text detected>"
27
+ # 2. Instruction to the model
28
+ instruction = f"Describe this image in one sentence, and include the visible text: '{visible_text}'."
29
+ # 3. Prepare inputs and generate
30
+ inputs = processor(images=image, text=instruction, return_tensors="pt").to(device)
31
  with torch.no_grad():
32
+ outputs = model.generate(**inputs, max_new_tokens=100)
33
+ desc = processor.decode(outputs[0], skip_special_tokens=True)
34
+ captions.append(desc.strip())
35
  return "\n".join([f"{i+1}. {cap}" for i, cap in enumerate(captions)])
36
 
37
+ # Build interactive Gradio interface
38
+ with gr.Blocks(title="Accurate Alt‑Text Generator with OCR") as demo:
39
+ gr.Markdown("# Advanced Alt‑Text Generator with OCR")
40
+ gr.Markdown(
41
+ "Upload multiple images, preview them below, and generate precise descriptions that include any on‑image text."
42
+ )
43
+ # File uploader & preview gallery
44
+ file_input = gr.File(
45
+ label="Upload Images",
46
+ file_count="multiple",
47
+ file_types=["image"]
48
+ )
49
+ preview = gr.Gallery(label="Image Preview").style(grid=[2,4])
50
+ # Controls
51
+ with gr.Row():
52
+ generate_btn = gr.Button("Generate Alt Text")
53
+ clear_btn = gr.Button("Clear All")
54
+ output = gr.Textbox(
55
+ label="Generated Alt Texts",
56
+ lines=10,
57
+ elem_id="output_box"
58
+ )
59
+ copy_btn = gr.Button("Copy to Clipboard")
60
+
61
+ # Bind events
62
+ file_input.upload(lambda files: files, inputs=file_input, outputs=preview)
63
+ generate_btn.click(fn=generate_alt_text, inputs=file_input, outputs=output)
64
+ clear_btn.click(lambda: ([], [], ""), inputs=None, outputs=[file_input, preview, output])
65
+ copy_btn.click(fn=None, inputs=None, outputs=None,
66
+ js="() => { navigator.clipboard.writeText(document.getElementById('output_box').value); }")
 
 
 
67
 
68
  if __name__ == "__main__":
69
+ demo.launch()