Workpam commited on
Commit
560e1cb
·
verified ·
1 Parent(s): 593d55c

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +43 -27
app.py CHANGED
@@ -1,5 +1,5 @@
1
  import gradio as gr
2
- from transformers import Blip2Processor, Blip2ForConditionalGeneration
3
  from PIL import Image
4
  import torch
5
  import easyocr
@@ -8,8 +8,7 @@ import easyocr
8
  device = "cuda" if torch.cuda.is_available() else "cpu"
9
  model_name = "Salesforce/blip2-flan-t5-base"
10
 
11
- # Ensure you have sentencepiece installed: pip install sentencepiece
12
- # Load BLIP-2 base model (balanced performance)
13
  processor = Blip2Processor.from_pretrained(model_name, use_fast=False)
14
  model = (
15
  Blip2ForConditionalGeneration.from_pretrained(
@@ -19,50 +18,67 @@ model = (
19
  .to(device)
20
  )
21
 
22
- # Initialize EasyOCR (CPU-friendly)
23
  ocr_reader = easyocr.Reader(['en'], gpu=(device == "cuda"))
 
 
 
 
 
 
24
 
25
  # --- Core Generation Function ---
26
  def generate_alt_text(files, max_tokens):
 
 
 
 
27
  captions = []
28
- for f in files:
29
- # Load image
30
  path = f.name if hasattr(f, 'name') else f
 
31
  try:
32
  image = Image.open(path).convert("RGB")
33
  except Exception as e:
34
  captions.append(f"Error opening {path}: {e}")
35
  continue
36
- # OCR extraction
37
  try:
38
  ocr_texts = ocr_reader.readtext(path, detail=0)
39
  visible_text = ' '.join(ocr_texts).strip()
40
  except Exception:
41
  visible_text = ''
42
- # BLIP2 captioning
43
- inputs = processor(
44
- images=image,
45
- text="Generate a concise and descriptive alt text for this image.",
46
- return_tensors="pt"
47
- ).to(device)
 
 
 
 
 
 
 
 
 
 
48
  outputs = model.generate(
49
  **inputs,
50
  max_new_tokens=int(max_tokens),
51
- num_beams=3,
52
  no_repeat_ngram_size=2
53
  )
54
  caption = processor.decode(outputs[0], skip_special_tokens=True).strip().capitalize()
55
- # Combine with OCR text if any
56
- if visible_text:
57
- caption = f"{caption} Text on image: '{visible_text}'."
58
  captions.append(caption)
59
- return captions
60
 
61
  # --- Gradio Interface ---
62
- with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
63
- gr.Markdown("# Balanced High‑End Alt Text Generator")
64
  gr.Markdown(
65
- "This app uses BLIP‑2 Base for high‑quality captions plus EasyOCR for visible text extraction."
66
  )
67
 
68
  with gr.Row():
@@ -75,7 +91,7 @@ with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
75
  minimum=16,
76
  maximum=128,
77
  step=16,
78
- value=32,
79
  label="Max Tokens for Caption"
80
  )
81
  generate_btn = gr.Button("Generate Alt Text")
@@ -88,8 +104,8 @@ with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
88
  show_copy_button=True
89
  )
90
 
91
- # Bindings
92
- file_input.upload(lambda files: files, inputs=file_input, outputs=gallery)
93
  generate_btn.click(
94
  fn=generate_alt_text,
95
  inputs=[file_input, max_tokens],
@@ -97,10 +113,10 @@ with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
97
  queue=True
98
  )
99
  clear_btn.click(
100
- lambda: ([], 32, []),
101
- inputs=None,
102
  outputs=[file_input, max_tokens, gallery, output]
103
  )
104
 
105
- # Launch
 
106
  demo.launch(server_name="0.0.0.0", server_port=7860)
 
1
  import gradio as gr
2
+ from transformers import Blip2Processor, Blip2ForConditionalGeneration, pipeline
3
  from PIL import Image
4
  import torch
5
  import easyocr
 
8
  device = "cuda" if torch.cuda.is_available() else "cpu"
9
  model_name = "Salesforce/blip2-flan-t5-base"
10
 
11
+ # Ensure sentencepiece is installed for T5 tokenizers
 
12
  processor = Blip2Processor.from_pretrained(model_name, use_fast=False)
13
  model = (
14
  Blip2ForConditionalGeneration.from_pretrained(
 
18
  .to(device)
19
  )
20
 
21
+ # Initialize EasyOCR (CPU-friendly; GPU if available)
22
  ocr_reader = easyocr.Reader(['en'], gpu=(device == "cuda"))
23
+ # Object detection with DETR for object context
24
+ detector = pipeline(
25
+ "object-detection",
26
+ model="facebook/detr-resnet-50",
27
+ device=0 if device == "cuda" else -1
28
+ )
29
 
30
  # --- Core Generation Function ---
31
  def generate_alt_text(files, max_tokens):
32
+ """
33
+ Generate detailed alt text for each image using BLIP-2, OCR, and object detection.
34
+ Returns a newline-separated string of captions.
35
+ """
36
  captions = []
37
+ for f in files or []:
 
38
  path = f.name if hasattr(f, 'name') else f
39
+ # Load image
40
  try:
41
  image = Image.open(path).convert("RGB")
42
  except Exception as e:
43
  captions.append(f"Error opening {path}: {e}")
44
  continue
45
+ # Extract OCR text
46
  try:
47
  ocr_texts = ocr_reader.readtext(path, detail=0)
48
  visible_text = ' '.join(ocr_texts).strip()
49
  except Exception:
50
  visible_text = ''
51
+ # Detect objects
52
+ try:
53
+ objs = detector(image)
54
+ # get unique labels, top-5
55
+ labels = list({o['label'] for o in objs[:5]})
56
+ objects_text = ', '.join(labels)
57
+ except Exception:
58
+ objects_text = ''
59
+ # Build prompt
60
+ prompt = "Generate a concise, descriptive alt text for this image."
61
+ if visible_text:
62
+ prompt += f" Include visible text: '{visible_text}'."
63
+ if objects_text:
64
+ prompt += f" Mention objects: {objects_text}."
65
+ # BLIP-2 captioning
66
+ inputs = processor(images=image, text=prompt, return_tensors="pt").to(device)
67
  outputs = model.generate(
68
  **inputs,
69
  max_new_tokens=int(max_tokens),
70
+ num_beams=4,
71
  no_repeat_ngram_size=2
72
  )
73
  caption = processor.decode(outputs[0], skip_special_tokens=True).strip().capitalize()
 
 
 
74
  captions.append(caption)
75
+ return "\n".join(captions) if captions else ""
76
 
77
  # --- Gradio Interface ---
78
+ with gr.Blocks(title="Refined High-End Alt Text Generator") as demo:
79
+ gr.Markdown("# Refined High‑End Alt Text Generator")
80
  gr.Markdown(
81
+ "This app uses BLIP‑2 Base for rich captions, EasyOCR for text extraction, and DETR for object context."
82
  )
83
 
84
  with gr.Row():
 
91
  minimum=16,
92
  maximum=128,
93
  step=16,
94
+ value=64,
95
  label="Max Tokens for Caption"
96
  )
97
  generate_btn = gr.Button("Generate Alt Text")
 
104
  show_copy_button=True
105
  )
106
 
107
+ # Bind events
108
+ file_input.upload(lambda files: files, file_input, gallery)
109
  generate_btn.click(
110
  fn=generate_alt_text,
111
  inputs=[file_input, max_tokens],
 
113
  queue=True
114
  )
115
  clear_btn.click(
116
+ lambda: ([], 64, [], ""),
 
117
  outputs=[file_input, max_tokens, gallery, output]
118
  )
119
 
120
+ demo.queue()
121
+
122
  demo.launch(server_name="0.0.0.0", server_port=7860)