Workpam commited on
Commit
f02a49a
·
verified ·
1 Parent(s): 6b986f3

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +88 -290
app.py CHANGED
@@ -1,316 +1,114 @@
1
  import os
2
- import gradio as gr
3
- import easyocr
4
  import torch
 
5
  from PIL import Image
6
- from concurrent.futures import ThreadPoolExecutor
7
- from transformers import BlipProcessor, BlipForConditionalGeneration
8
  from functools import lru_cache
9
- from spellchecker import SpellChecker # Import for spell checking
 
10
 
11
- # 1. Configuration and Global Variables
12
  class Config:
13
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
14
- BLIP_MODEL_NAME = "Salesforce/blip-image-captioning-base"
15
- YOLO_MODEL_NAME = "yolov5s"
16
- EASYOCR_LANGUAGES = ["en"]
17
- MAX_CAPTION_LENGTH = 75
18
- MIN_CAPTION_LENGTH = 5
19
- NUM_BLIP_BEAMS = 5
20
- NO_REPEAT_NGRAM_SIZE = 2
21
- CPU_CORES = os.cpu_count() or 4
22
-
23
- # 2. Model Loading (with caching for efficiency)
24
  @lru_cache(maxsize=1)
25
- def load_blip_model():
26
- """Loads the BLIP model and processor."""
27
- print(f"Loading BLIP model '{Config.BLIP_MODEL_NAME}' to {Config.DEVICE}...")
28
- processor = BlipProcessor.from_pretrained(Config.BLIP_MODEL_NAME)
29
- model = BlipForConditionalGeneration.from_pretrained(Config.BLIP_MODEL_NAME).to(Config.DEVICE)
30
- print("BLIP model loaded.")
31
  return processor, model
32
 
33
  @lru_cache(maxsize=1)
34
- def load_ocr_reader():
35
- """Initializes EasyOCR reader."""
36
- print(f"Initializing EasyOCR reader for languages {Config.EASYOCR_LANGUAGES} (GPU: {torch.cuda.is_available()})...")
37
- return easyocr.Reader(Config.EASYOCR_LANGUAGES, gpu=torch.cuda.is_available())
38
-
39
- @lru_cache(maxsize=1)
40
- def load_yolo_model():
41
- """Optionally loads YOLOv5 for object detection."""
42
- try:
43
- from ultralytics import YOLO
44
- print(f"Loading YOLO model '{Config.YOLO_MODEL_NAME}'...")
45
- yolo_model = YOLO(Config.YOLO_MODEL_NAME)
46
- if torch.cuda.is_available():
47
- yolo_model.to(Config.DEVICE)
48
- print("YOLO model loaded.")
49
- return yolo_model
50
- except ImportError:
51
- print("YOLOv5 (ultralytics library) not installed. Object detection will be disabled.")
52
- return None
53
- except Exception as e:
54
- print(f"Error loading YOLO model: {e}. Object detection will be disabled.")
55
- return None
56
-
57
- @lru_cache(maxsize=1)
58
- def load_spell_checker():
59
- """Loads a spell checker."""
60
- print("Loading spell checker...")
61
- return SpellChecker()
62
-
63
- # Load models once at startup
64
- blip_processor, blip_model = load_blip_model()
65
- ocr_reader = load_ocr_reader()
66
- yolo_model = load_yolo_model()
67
- use_yolo = yolo_model is not None
68
- spell = load_spell_checker() # Initialize spell checker
69
-
70
- # Helper function for OCR text cleanup
71
- def clean_ocr_text(text):
72
- if not text:
73
- return ""
74
-
75
- # 1. Basic Spell Correction
76
- words = text.split()
77
- corrected_words = [spell.correction(word) if spell.correction(word) is not None else word for word in words]
78
- cleaned_text = ' '.join(corrected_words)
79
-
80
- # 2. Case Normalization (e.g., sentence case)
81
- cleaned_text = cleaned_text.lower() # Convert to lowercase first
82
- if cleaned_text:
83
- cleaned_text = cleaned_text[0].upper() + cleaned_text[1:] # Capitalize first letter
84
-
85
- # 3. Add punctuation (simple heuristic: add a period if missing)
86
- if cleaned_text and not cleaned_text.endswith(('.', '!', '?')):
87
- cleaned_text += '.'
88
-
89
- # Remove excessive spaces
90
- cleaned_text = ' '.join(cleaned_text.split())
91
-
92
- return cleaned_text
93
-
94
- # 3. Core Processing Function
95
- def process_single_image(image_path, max_alt_text_len):
96
- """
97
- Processes a single image to generate its alt text.
98
- Handles image loading, OCR, object detection, and BLIP captioning.
99
- """
100
- filename = os.path.basename(image_path)
101
- print(f"Processing image: {filename}")
102
-
103
- try:
104
- image = Image.open(image_path).convert('RGB')
105
- image.verify()
106
- except Exception as e:
107
- print(f"Error opening or verifying file {filename}: {e}")
108
- # Return only the error message, not filename in output string
109
- return filename, f"<Error opening or verifying file: {e}>"
110
-
111
- blip_caption = ""
112
- object_labels = ""
113
- ocr_text = ""
114
-
115
- # BLIP captioning
116
- try:
117
- inputs = blip_processor(images=image, return_tensors="pt").to(Config.DEVICE)
118
- output_ids = blip_model.generate(
119
- **inputs,
120
- num_beams=Config.NUM_BLIP_BEAMS,
121
- max_length=Config.MAX_CAPTION_LENGTH,
122
- min_length=Config.MIN_CAPTION_LENGTH,
123
- no_repeat_ngram_size=Config.NO_REPEAT_NGRAM_SIZE
124
  )
125
- blip_caption = blip_processor.decode(output_ids[0], skip_special_tokens=True).strip()
126
- # Capitalize BLIP caption for better sentence structure
127
- if blip_caption:
128
- blip_caption = blip_caption[0].upper() + blip_caption[1:]
129
- if not blip_caption.endswith(('.', '!', '?')):
130
- blip_caption += '.'
131
- except Exception as e:
132
- print(f"Error generating BLIP caption for {filename}: {e}")
133
- blip_caption = f"Error generating primary caption."
134
-
135
- # OCR and Cleanup
136
- try:
137
- texts = ocr_reader.readtext(image_path, detail=0)
138
- raw_ocr = ' '.join(texts).strip()
139
- ocr_text = clean_ocr_text(raw_ocr) # Apply cleanup
140
- except Exception as e:
141
- print(f"Error performing OCR for {filename}: {e}")
142
- ocr_text = ''
143
-
144
- # Object detection
145
- if use_yolo:
146
- try:
147
- results = yolo_model(image_path)
148
- if results and len(results) > 0:
149
- df = results[0].pandas().xyxy[0]
150
- labels_list = df['name'].unique().tolist()[:5]
151
- object_labels = ', '.join(labels_list)
152
- # Capitalize object labels for consistency
153
- if object_labels:
154
- object_labels = object_labels[0].upper() + object_labels[1:]
155
- else:
156
- object_labels = ''
157
- except Exception as e:
158
- print(f"Error performing object detection for {filename}: {e}")
159
- object_labels = ''
160
-
161
- # Assemble alt text
162
- parts = []
163
- # Always start with BLIP caption as the primary description
164
- if blip_caption:
165
- parts.append(blip_caption)
166
-
167
- # Add objects if detected
168
- if object_labels:
169
- parts.append(f"Objects detected: {object_labels}.") # More natural phrasing
170
-
171
- # Add cleaned OCR text if found
172
- if ocr_text:
173
- parts.append(f"Visible text: \"{ocr_text}\"") # Clearly distinguish OCR text
174
-
175
- alt_text = ' '.join(filter(None, parts)) # Join with space, not semicolon, for better flow
176
-
177
- # Final truncation, ensuring word boundaries and ellipsis
178
- if len(alt_text) > max_alt_text_len:
179
- truncated_text = alt_text[:max_alt_text_len]
180
- last_space_index = truncated_text.rfind(' ')
181
-
182
- if last_space_index != -1 and last_space_index > max_alt_text_len * 0.8: # Ensure meaningful truncation
183
- alt_text = truncated_text[:last_space_index].strip() + '…'
184
- else:
185
- alt_text = truncated_text.strip() + '…' # Fallback for short texts or no spaces
186
-
187
- # Remove any trailing periods before adding ellipsis if truncation happened
188
- if alt_text.endswith('…') and alt_text.endswith('.…'):
189
- alt_text = alt_text[:-2] + '…'
190
-
191
- print(f"Finished processing {filename}. Alt text length: {len(alt_text)}")
192
- return filename, alt_text # Keep filename in this return for internal processing
193
-
194
- def generate_alt_text_for_files(file_objs, max_alt_text_len):
195
- """
196
- Generates alt text for multiple image files in parallel.
197
- Removes the filename from the final displayed output.
198
- """
199
- if not file_objs:
200
- return "No files uploaded."
201
-
202
- image_paths = [getattr(f, 'name', f) for f in file_objs]
203
- print(f"Generating alt text for {len(image_paths)} files.")
204
-
205
- max_workers = min(len(image_paths), Config.CPU_CORES)
206
-
207
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
208
- results = list(executor.map(lambda p: process_single_image(p, max_alt_text_len), image_paths))
209
-
210
- # FORMATTING THE OUTPUT: Exclude filename from the final display string
211
- lines = [f"{txt}" for fn, txt in results] # Only the generated text, not filename
212
- return "\n".join(lines)
213
-
214
- # 4. Gradio Interface (UI/UX Improvements)
215
- with gr.Blocks(title="AI-Powered Alt Text Generator", theme=gr.themes.Soft()) as app:
216
- gr.Markdown("# ✨ AI-Powered Alt Text Generator ✨")
217
  gr.Markdown(
218
- "Upload images to generate rich alt text using **BLIP** for captions, "
219
- "**EasyOCR** for visible text (with basic spell correction), and optional **YOLOv5** for object detection. "
220
- "Models are loaded once for faster subsequent processing."
221
  )
222
-
223
  with gr.Row():
224
- with gr.Column(scale=2):
225
  file_input = gr.File(
226
  label="Upload Images",
227
  file_count="multiple",
228
- file_types=["image"],
229
- interactive=True,
230
- height=200
231
  )
232
- max_len_slider = gr.Slider(
233
- 50, 500, step=10, value=250,
234
- label="Maximum Alt Text Length (characters)",
235
- info="Truncates the generated alt text to this length, preserving word boundaries.",
236
- interactive=True
 
237
  )
238
-
239
- with gr.Row():
240
- gen_btn = gr.Button("🚀 Generate Alt Text", variant="primary")
241
- copy_btn = gr.Button("📋 Copy to Clipboard", variant="secondary")
242
-
243
- with gr.Column(scale=3):
244
  gallery = gr.Gallery(
245
- label="Uploaded Image Previews",
246
- columns=4,
247
- rows=1,
248
- object_fit="contain",
249
- height="auto"
250
  )
251
- output_text = gr.Textbox(
252
- label="Generated Alt Text",
253
- lines=15,
254
- interactive=False,
255
- elem_id="output_text",
256
- show_copy_button=True
257
- )
258
-
259
- file_input.change(fn=lambda files: files, inputs=file_input, outputs=gallery)
260
-
261
- gen_btn.click(
262
- fn=generate_alt_text_for_files,
263
- inputs=[file_input, max_len_slider],
264
- outputs=[output_text],
265
- queue=True,
266
  )
267
 
268
- copy_btn.click(
269
- None,
270
- inputs=None,
271
- outputs=None,
272
- js="""
273
- () => {
274
- const outputElement = document.getElementById('output_text');
275
- if (outputElement) {
276
- if (navigator.clipboard && navigator.clipboard.writeText) {
277
- navigator.clipboard.writeText(outputElement.value)
278
- .then(() => alert('Alt text copied to clipboard!'))
279
- .catch(err => console.error('Failed to copy text: ', err));
280
- } else {
281
- outputElement.select();
282
- document.execCommand('copy');
283
- alert('Alt text copied to clipboard!');
284
- }
285
- } else {
286
- alert('Output text element not found.');
287
- }
288
- }
289
- """
290
- )
291
-
292
- gr.Markdown(
293
- """
294
- ---
295
- **How it works:**
296
- - **BLIP:** Generates a descriptive caption of the image's content.
297
- - **EasyOCR:** Detects and extracts visible text within the image, then applies basic spell correction and formatting.
298
- - **YOLOv5 (Optional):** Identifies and lists prominent objects. (Requires `ultralytics` library)
299
-
300
- This tool is a demonstration and may not be perfect for all images.
301
- For optimal performance, ensure you have a GPU (CUDA) enabled.
302
- """
303
- )
304
-
305
- def main():
306
- print(f"\nStarting Gradio app on device: {Config.DEVICE}")
307
- print(f"YOLO enabled: {use_yolo}\n")
308
- app.launch(
309
- server_name="0.0.0.0",
310
- server_port=7860,
311
- share=True,
312
- debug=True
313
  )
314
 
315
- if __name__ == "__main__":
316
- main()
 
 
 
 
 
 
1
  import os
 
 
2
  import torch
3
+ import easyocr
4
  from PIL import Image
 
 
5
  from functools import lru_cache
6
+ import gradio as gr
7
+ from transformers import Blip2Processor, Blip2ForConditionalGeneration
8
 
9
+ # --- Configuration ---
10
  class Config:
11
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
12
+ BLIP2_MODEL = "Salesforce/blip2-flan-t5-xl" # High-end multilingual model
13
+ OCR_LANGS = ["en"] # Add other language codes if needed
14
+ MAX_LENGTH = 128
15
+ MIN_LENGTH = 10
16
+ BEAMS = 4
17
+ NO_REPEAT_NGRAM_SIZE = 3
18
+
19
+ # --- Lazy Model Loaders ---
 
 
20
  @lru_cache(maxsize=1)
21
+ def get_blip2_model():
22
+ processor = Blip2Processor.from_pretrained(Config.BLIP2_MODEL)
23
+ model = Blip2ForConditionalGeneration.from_pretrained(
24
+ Config.BLIP2_MODEL,
25
+ torch_dtype=torch.float16 if Config.DEVICE == "cuda" else torch.float32
26
+ ).to(Config.DEVICE)
27
  return processor, model
28
 
29
  @lru_cache(maxsize=1)
30
+ def get_ocr_reader():
31
+ return easyocr.Reader(Config.OCR_LANGS, gpu=(Config.DEVICE == "cuda"))
32
+
33
+ # --- Core Processing ---
34
+ def generate_alt_text(images, max_len):
35
+ processor, model = get_blip2_model()
36
+ ocr_reader = get_ocr_reader()
37
+ results = []
38
+ for img in images:
39
+ # Convert file-like to PIL image
40
+ image = Image.open(img.name if hasattr(img, 'name') else img).convert("RGB")
41
+ # 1) OCR extraction
42
+ text_segments = ocr_reader.readtext(
43
+ img.name if hasattr(img, 'name') else img, detail=0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
44
  )
45
+ visible_text = ' '.join(text_segments).strip()
46
+ # 2) BLIP2 captioning
47
+ inputs = processor(images=image, text="Generate a detailed alt text for this image.", return_tensors="pt").to(Config.DEVICE)
48
+ with torch.no_grad():
49
+ outputs = model.generate(
50
+ **inputs,
51
+ max_new_tokens=int(max_len),
52
+ min_length=Config.MIN_LENGTH,
53
+ num_beams=Config.BEAMS,
54
+ no_repeat_ngram_size=Config.NO_REPEAT_NGRAM_SIZE
55
+ )
56
+ caption = processor.decode(outputs[0], skip_special_tokens=True).strip()
57
+ # 3) Combine with OCR text if present
58
+ if visible_text:
59
+ caption = f"{caption} Visible text: '{visible_text}'."
60
+ results.append(caption)
61
+ return { 'captions': results }
62
+
63
+ # --- Gradio Interface ---
64
+ with gr.Blocks(title="🔍 High-End Alt Text Generator") as app:
65
+ gr.Markdown("# High‑End AI‑Powered Alt Text Generator")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
66
  gr.Markdown(
67
+ "Upload images and get descriptive, context‑aware alt text generated by BLIP‑2 XL, "
68
+ "augmented with OCR‑extracted text when available."
 
69
  )
 
70
  with gr.Row():
71
+ with gr.Column(scale=1):
72
  file_input = gr.File(
73
  label="Upload Images",
74
  file_count="multiple",
75
+ file_types=["image"]
 
 
76
  )
77
+ max_len = gr.Slider(
78
+ minimum=32,
79
+ maximum=Config.MAX_LENGTH,
80
+ step=8,
81
+ value=64,
82
+ label="Max Tokens for Caption"
83
  )
84
+ run_btn = gr.Button("Generate Alt Text", variant="primary")
85
+ with gr.Column(scale=1):
 
 
 
 
86
  gallery = gr.Gallery(
87
+ label="Image Preview",
88
+ columns=3,
89
+ rows=2,
90
+ interactive=False
 
91
  )
92
+ output = gr.Textbox(
93
+ label="Generated Alt Texts",
94
+ lines=10,
95
+ interactive=False,
96
+ show_copy_button=True
 
 
 
 
 
 
 
 
 
 
97
  )
98
 
99
+ # Bindings
100
+ file_input.upload(lambda files: files, inputs=file_input, outputs=gallery)
101
+ run_btn.click(
102
+ fn=generate_alt_text,
103
+ inputs=[file_input, max_len],
104
+ outputs=output,
105
+ queue=True
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
106
  )
107
 
108
+ app.queue()
109
+ # Launch
110
+ app.launch(
111
+ server_name="0.0.0.0",
112
+ server_port=7860,
113
+ share=True
114
+ )