Workpam commited on
Commit
a656b42
·
verified ·
1 Parent(s): 40fff6e

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +30 -44
app.py CHANGED
@@ -6,10 +6,8 @@ import easyocr
6
 
7
  # --- Configuration ---
8
  device = "cuda" if torch.cuda.is_available() else "cpu"
9
- # Use a valid BLIP-2 model identifier
10
  model_name = "Salesforce/blip2-flan-t5-xl"
11
 
12
- # Ensure sentencepiece is installed: pip install sentencepiece
13
  processor = Blip2Processor.from_pretrained(model_name, use_fast=False)
14
  model = (
15
  Blip2ForConditionalGeneration.from_pretrained(
@@ -19,9 +17,7 @@ model = (
19
  .to(device)
20
  )
21
 
22
- # Initialize EasyOCR (CPU-friendly)
23
  ocr_reader = easyocr.Reader(['en'], gpu=(device == "cuda"))
24
- # Object detection with DETR for object context
25
  detector = pipeline(
26
  "object-detection",
27
  model="facebook/detr-resnet-50",
@@ -32,34 +28,33 @@ detector = pipeline(
32
  def generate_alt_text(files, max_tokens):
33
  captions = []
34
  for f in files or []:
35
- path = f.name if hasattr(f, 'name') else f
36
- # Load image
37
- try:
38
- image = Image.open(path).convert("RGB")
39
- except Exception as e:
40
- captions.append(f"Error opening {path}: {e}")
41
- continue
42
- # OCR extraction
43
  try:
44
- ocr_texts = ocr_reader.readtext(path, detail=0)
45
- visible_text = ' '.join(ocr_texts).strip()
46
  except Exception:
47
  visible_text = ''
48
- # Detect objects
 
49
  try:
50
- objs = detector(image)
51
  labels = list({o['label'] for o in objs[:5]})
52
  objects_text = ', '.join(labels)
53
  except Exception:
54
  objects_text = ''
55
- # Build prompt
 
56
  prompt = "Generate a concise, descriptive alt text for this image."
57
  if visible_text:
58
  prompt += f" Include visible text: '{visible_text}'."
59
  if objects_text:
60
  prompt += f" Mention objects: {objects_text}."
61
- # BLIP-2 captioning
62
- inputs = processor(images=image, text=prompt, return_tensors="pt").to(device)
 
63
  outputs = model.generate(
64
  **inputs,
65
  max_new_tokens=int(max_tokens),
@@ -68,48 +63,39 @@ def generate_alt_text(files, max_tokens):
68
  )
69
  caption = processor.decode(outputs[0], skip_special_tokens=True).strip().capitalize()
70
  captions.append(caption)
71
- return "\n".join(captions)
 
72
 
73
  # --- Gradio Interface ---
74
- with gr.Blocks(title="Refined High-End Alt Text Generator") as demo:
75
  gr.Markdown("# Refined High‑End Alt Text Generator")
76
- gr.Markdown(
77
- "This app uses BLIP‑2 XL for rich captions, EasyOCR for text extraction, and DETR for object context."
78
- )
79
 
80
  with gr.Row():
81
- file_input = gr.File(
82
- label="Upload Images",
83
- file_count="multiple",
84
- file_types=["image"]
85
- )
86
- max_tokens = gr.Slider(
87
- minimum=16,
88
- maximum=128,
89
- step=16,
90
- value=64,
91
- label="Max Tokens for Caption"
92
- )
93
  generate_btn = gr.Button("Generate Alt Text")
94
  clear_btn = gr.Button("Clear All")
95
 
96
  gallery = gr.Gallery(label="Image Preview", columns=4)
97
- output = gr.Textbox(
98
- label="Generated Alt Texts",
99
- lines=10,
100
- show_copy_button=True
 
 
 
101
  )
102
 
103
- # Bind events correctly using inputs/outputs lists
104
- file_input.upload(lambda files: files, inputs=[file_input], outputs=[gallery])
105
  generate_btn.click(
106
  fn=generate_alt_text,
107
  inputs=[file_input, max_tokens],
108
- outputs=[output],
109
  queue=True
110
  )
 
111
  clear_btn.click(
112
- lambda: ([], 64, [], ""),
113
  outputs=[file_input, max_tokens, gallery, output]
114
  )
115
 
 
6
 
7
  # --- Configuration ---
8
  device = "cuda" if torch.cuda.is_available() else "cpu"
 
9
  model_name = "Salesforce/blip2-flan-t5-xl"
10
 
 
11
  processor = Blip2Processor.from_pretrained(model_name, use_fast=False)
12
  model = (
13
  Blip2ForConditionalGeneration.from_pretrained(
 
17
  .to(device)
18
  )
19
 
 
20
  ocr_reader = easyocr.Reader(['en'], gpu=(device == "cuda"))
 
21
  detector = pipeline(
22
  "object-detection",
23
  model="facebook/detr-resnet-50",
 
28
  def generate_alt_text(files, max_tokens):
29
  captions = []
30
  for f in files or []:
31
+ # load image
32
+ img = Image.open(f.name).convert("RGB")
33
+
34
+ # OCR
 
 
 
 
35
  try:
36
+ txts = ocr_reader.readtext(f.name, detail=0)
37
+ visible_text = ' '.join(txts).strip()
38
  except Exception:
39
  visible_text = ''
40
+
41
+ # object context
42
  try:
43
+ objs = detector(img)
44
  labels = list({o['label'] for o in objs[:5]})
45
  objects_text = ', '.join(labels)
46
  except Exception:
47
  objects_text = ''
48
+
49
+ # build prompt
50
  prompt = "Generate a concise, descriptive alt text for this image."
51
  if visible_text:
52
  prompt += f" Include visible text: '{visible_text}'."
53
  if objects_text:
54
  prompt += f" Mention objects: {objects_text}."
55
+
56
+ # BLIP‑2 captioning
57
+ inputs = processor(images=img, text=prompt, return_tensors="pt").to(device)
58
  outputs = model.generate(
59
  **inputs,
60
  max_new_tokens=int(max_tokens),
 
63
  )
64
  caption = processor.decode(outputs[0], skip_special_tokens=True).strip().capitalize()
65
  captions.append(caption)
66
+
67
+ return "\n\n".join(captions)
68
 
69
  # --- Gradio Interface ---
70
+ with gr.Blocks(title="Refined High‑End Alt Text Generator") as demo:
71
  gr.Markdown("# Refined High‑End Alt Text Generator")
72
+ gr.Markdown("Uses BLIP‑2 XL for captions, EasyOCR for text extraction, and DETR for object context.")
 
 
73
 
74
  with gr.Row():
75
+ file_input = gr.File(label="Upload Images", file_count="multiple", file_types=["image"])
76
+ max_tokens = gr.Slider(16, 128, value=64, step=16, label="Max Tokens for Caption")
 
 
 
 
 
 
 
 
 
 
77
  generate_btn = gr.Button("Generate Alt Text")
78
  clear_btn = gr.Button("Clear All")
79
 
80
  gallery = gr.Gallery(label="Image Preview", columns=4)
81
+ output = gr.Textbox(label="Generated Alt Texts", lines=10, show_copy_button=True)
82
+
83
+ # update gallery when files are selected
84
+ file_input.change(
85
+ fn=lambda files: [Image.open(f.name).convert("RGB") for f in (files or [])],
86
+ inputs=file_input,
87
+ outputs=gallery
88
  )
89
 
 
 
90
  generate_btn.click(
91
  fn=generate_alt_text,
92
  inputs=[file_input, max_tokens],
93
+ outputs=output,
94
  queue=True
95
  )
96
+
97
  clear_btn.click(
98
+ fn=lambda: ([], 64, [], ""),
99
  outputs=[file_input, max_tokens, gallery, output]
100
  )
101