Workpam commited on
Commit
94bc185
·
verified ·
1 Parent(s): 47fe535

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +222 -101
app.py CHANGED
@@ -5,127 +5,248 @@ import torch
5
  from PIL import Image
6
  from concurrent.futures import ThreadPoolExecutor
7
  from transformers import BlipProcessor, BlipForConditionalGeneration
8
-
9
- # 1. Initialize device
10
- device = "cuda" if torch.cuda.is_available() else "cpu"
11
-
12
- # 2. Load BLIP model and processor locally
13
- def load_blip():
14
- processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
15
- model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base").to(device)
 
 
 
 
 
 
 
 
 
 
 
 
 
16
  return processor, model
17
- processor, model = load_blip()
18
-
19
- # 3. Initialize EasyOCR
20
- gpu_flag = torch.cuda.is_available()
21
- ocr_reader = easyocr.Reader(["en"], gpu=gpu_flag)
22
-
23
- # 4. Optionally load YOLOv5 for object detection
24
- use_yolo = True
25
- yolo_model = None
26
- try:
27
- from ultralytics import YOLO
28
- yolo_model = YOLO("yolov5s")
29
- if gpu_flag:
30
- yolo_model.to(device)
31
- except Exception:
32
- use_yolo = False
33
 
34
- # 5. Determine CPU cores for parallelism
35
- cpu_cores = os.cpu_count() or 4
 
 
36
 
37
-
38
- def process_file(file_obj, max_len):
39
- path = getattr(file_obj, 'name', file_obj)
40
- filename = os.path.basename(path)
41
-
42
- # Verify image
43
  try:
44
- with Image.open(path).convert('RGB') as img:
45
- img.verify()
46
- except Exception:
47
- return filename, "<Error opening file>"
48
-
49
- # OCR
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  try:
51
- texts = ocr_reader.readtext(path, detail=0)
52
- visible = ' '.join(texts).strip()
53
- except Exception:
54
- visible = ''
 
55
 
56
- # Object detection
57
- labels = ''
58
- if use_yolo and yolo_model:
59
- try:
60
- results = yolo_model(path)
61
- df = results.pandas().xyxy[0]
62
- labels_list = df['name'].unique().tolist()[:5]
63
- labels = ', '.join(labels_list)
64
- except Exception:
65
- labels = ''
66
-
67
- # BLIP caption\
68
  try:
69
- image = Image.open(path).convert('RGB')
70
- inputs = processor(images=image, return_tensors="pt").to(device)
71
- output_ids = model.generate(
72
  **inputs,
73
- num_beams=5,
74
- max_length=75,
75
- min_length=5,
76
- no_repeat_ngram_size=2
77
  )
78
- base = processor.decode(output_ids[0], skip_special_tokens=True)
79
  except Exception as e:
80
- return filename, f"Error generating caption: {e}"
81
-
82
- # Assemble and truncate
83
- parts = [base.strip()]
84
- if labels:
85
- parts.append(f"Objects: {labels}")
86
- if visible:
87
- parts.append(f"Text: {visible}")
88
- caption = '; '.join(parts)
89
- if len(caption) > max_len:
90
- caption = caption[:max_len].rsplit(' ', 1)[0] + '…'
91
-
92
- return filename, caption
93
 
 
 
 
 
 
 
 
94
 
95
- def generate_alt_text(files, max_len):
96
- if not files:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  return ""
98
- max_workers = min(len(files), cpu_cores)
 
 
 
 
 
 
 
 
99
  with ThreadPoolExecutor(max_workers=max_workers) as executor:
100
- results = executor.map(lambda f: process_file(f, max_len), files)
 
 
 
101
  lines = [f"{fn}: {txt}" for fn, txt in results]
102
  return "\n".join(lines)
103
 
104
- # Build Gradio interface
105
- app = gr.Blocks(title="AI-Powered Alt Text Generator")
106
- with app:
107
- gr.Markdown("# AI-Powered Alt Text Generator")
108
- gr.Markdown("Generate rich alt text with OCR, optional object detection, and BLIP captions.")
109
 
110
  with gr.Row():
111
- file_input = gr.File(label="Upload Images", file_count="multiple", file_types=["image"])
112
- max_len = gr.Slider(50, 200, step=5, value=125, label="Max Alt Text Length (chars)")
113
-
114
- gallery = gr.Gallery(label="Preview", columns=4)
115
-
116
- with gr.Row():
117
- gen_btn = gr.Button("Generate Alt Text")
118
- copy_btn = gr.Button("Copy to Clipboard")
119
-
120
- output_text = gr.Textbox(label="Generated Alt Text", lines=10, interactive=False, elem_id="output_text")
121
-
122
- file_input.upload(lambda files: files, inputs=file_input, outputs=gallery)
123
- gen_btn.click(fn=generate_alt_text, inputs=[file_input, max_len], outputs=[output_text])
124
- copy_btn.click(None, None, None, js="() => { navigator.clipboard.writeText(document.getElementById('output_text').value); }")
125
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
126
  # For local testing
127
  def main():
128
- app.launch(server_name="0.0.0.0", server_port=7860, share=True)
 
 
 
 
 
129
 
130
  if __name__ == "__main__":
131
- main()
 
5
  from PIL import Image
6
  from concurrent.futures import ThreadPoolExecutor
7
  from transformers import BlipProcessor, BlipForConditionalGeneration
8
+ from functools import lru_cache # For caching models
9
+
10
+ # 1. Configuration and Global Variables
11
+ # Centralize configuration for easier modification
12
+ class Config:
13
+ DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
14
+ BLIP_MODEL_NAME = "Salesforce/blip-image-captioning-base"
15
+ YOLO_MODEL_NAME = "yolov5s" # Using yolov5s for faster inference
16
+ EASYOCR_LANGUAGES = ["en"]
17
+ MAX_CAPTION_LENGTH = 75 # Max length for BLIP output
18
+ MIN_CAPTION_LENGTH = 5 # Min length for BLIP output
19
+ NUM_BLIP_BEAMS = 5
20
+ NO_REPEAT_NGRAM_SIZE = 2
21
+ CPU_CORES = os.cpu_count() or 4 # Fallback to 4 if os.cpu_count() is None
22
+
23
+ # 2. Model Loading (with caching for efficiency)
24
+ @lru_cache(maxsize=1) # Cache the BLIP model and processor
25
+ def load_blip_model():
26
+ """Loads the BLIP model and processor."""
27
+ processor = BlipProcessor.from_pretrained(Config.BLIP_MODEL_NAME)
28
+ model = BlipForConditionalGeneration.from_pretrained(Config.BLIP_MODEL_NAME).to(Config.DEVICE)
29
  return processor, model
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
 
31
+ @lru_cache(maxsize=1) # Cache the EasyOCR reader
32
+ def load_ocr_reader():
33
+ """Initializes EasyOCR reader."""
34
+ return easyocr.Reader(Config.EASYOCR_LANGUAGES, gpu=torch.cuda.is_available())
35
 
36
+ @lru_cache(maxsize=1) # Cache the YOLO model
37
+ def load_yolo_model():
38
+ """Optionally loads YOLOv5 for object detection."""
 
 
 
39
  try:
40
+ from ultralytics import YOLO
41
+ yolo_model = YOLO(Config.YOLO_MODEL_NAME)
42
+ if torch.cuda.is_available():
43
+ yolo_model.to(Config.DEVICE)
44
+ return yolo_model
45
+ except ImportError:
46
+ print("YOLOv5 not installed or 'ultralytics' not found. Object detection will be disabled.")
47
+ return None
48
+ except Exception as e:
49
+ print(f"Error loading YOLO model: {e}. Object detection will be disabled.")
50
+ return None
51
+
52
+ # Load models once at startup
53
+ blip_processor, blip_model = load_blip_model()
54
+ ocr_reader = load_ocr_reader()
55
+ yolo_model = load_yolo_model()
56
+ use_yolo = yolo_model is not None # Flag to check if YOLO is loaded
57
+
58
+ # 3. Core Processing Function
59
+ def process_single_image(image_path, max_alt_text_len):
60
+ """
61
+ Processes a single image to generate its alt text.
62
+ Handles image loading, OCR, object detection, and BLIP captioning.
63
+ """
64
+ filename = os.path.basename(image_path)
65
+
66
  try:
67
+ # Open and convert image once
68
+ image = Image.open(image_path).convert('RGB')
69
+ image.verify() # Verify image integrity
70
+ except Exception as e:
71
+ return filename, f"<Error opening or verifying file: {e}>"
72
 
73
+ # Initialize results
74
+ blip_caption = ""
75
+ object_labels = ""
76
+ ocr_text = ""
77
+
78
+ # BLIP captioning (can be done in parallel with OCR/YOLO for speed if desired, but here sequential)
 
 
 
 
 
 
79
  try:
80
+ inputs = blip_processor(images=image, return_tensors="pt").to(Config.DEVICE)
81
+ output_ids = blip_model.generate(
 
82
  **inputs,
83
+ num_beams=Config.NUM_BLIP_BEAMS,
84
+ max_length=Config.MAX_CAPTION_LENGTH,
85
+ min_length=Config.MIN_CAPTION_LENGTH,
86
+ no_repeat_ngram_size=Config.NO_REPEAT_NGRAM_SIZE
87
  )
88
+ blip_caption = blip_processor.decode(output_ids[0], skip_special_tokens=True).strip()
89
  except Exception as e:
90
+ print(f"Error generating BLIP caption for {filename}: {e}")
91
+ blip_caption = f"Error generating caption: {e}" # Include error in caption if it's the main issue
 
 
 
 
 
 
 
 
 
 
 
92
 
93
+ # OCR
94
+ try:
95
+ texts = ocr_reader.readtext(image_path, detail=0)
96
+ ocr_text = ' '.join(texts).strip()
97
+ except Exception as e:
98
+ print(f"Error performing OCR for {filename}: {e}")
99
+ ocr_text = '' # Keep empty if OCR fails
100
 
101
+ # Object detection
102
+ if use_yolo:
103
+ try:
104
+ results = yolo_model(image_path)
105
+ # YOLOv5's new API might require different access for pandas DataFrame
106
+ # Assuming results[0].pandas().xyxy[0] is still valid for a single image result
107
+ df = results[0].pandas().xyxy[0]
108
+ labels_list = df['name'].unique().tolist()[:5] # Limit to top 5 unique objects
109
+ object_labels = ', '.join(labels_list)
110
+ except Exception as e:
111
+ print(f"Error performing object detection for {filename}: {e}")
112
+ object_labels = '' # Keep empty if object detection fails
113
+
114
+ # Assemble and truncate alt text
115
+ parts = [blip_caption]
116
+ if object_labels:
117
+ parts.append(f"Objects: {object_labels}")
118
+ if ocr_text:
119
+ parts.append(f"Text: {ocr_text}")
120
+
121
+ alt_text = '; '.join(filter(None, parts)) # Filter out empty strings before joining
122
+
123
+ # Truncate to max_alt_text_len, respecting word boundaries
124
+ if len(alt_text) > max_alt_text_len:
125
+ alt_text = alt_text[:max_alt_text_len].rsplit(' ', 1)[0]
126
+ if not alt_text.endswith('…'): # Add ellipsis if not already present
127
+ alt_text += '…'
128
+
129
+ return filename, alt_text
130
+
131
+ def generate_alt_text_for_files(file_objs, max_alt_text_len):
132
+ """
133
+ Generates alt text for multiple image files in parallel.
134
+ """
135
+ if not file_objs:
136
  return ""
137
+
138
+ image_paths = [getattr(f, 'name', f) for f in file_objs]
139
+
140
+ # Use ThreadPoolExecutor for concurrent processing
141
+ # The number of workers can be tuned, but Config.CPU_CORES is a good starting point.
142
+ # For GPU-heavy tasks like BLIP, too many threads might saturate the GPU.
143
+ # Consider adjusting based on your specific GPU/CPU setup.
144
+ max_workers = min(len(image_paths), Config.CPU_CORES)
145
+
146
  with ThreadPoolExecutor(max_workers=max_workers) as executor:
147
+ # Map the single image processing function to each file path
148
+ results = executor.map(lambda p: process_single_image(p, max_alt_text_len), image_paths)
149
+
150
+ # Format the results for display
151
  lines = [f"{fn}: {txt}" for fn, txt in results]
152
  return "\n".join(lines)
153
 
154
+ # 4. Gradio Interface (UI/UX Improvements)
155
+ with gr.Blocks(title="AI-Powered Alt Text Generator", theme=gr.themes.Soft()) as app: # Added a theme
156
+ gr.Markdown("# ✨ AI-Powered Alt Text Generator ✨")
157
+ gr.Markdown("Upload images to generate rich alt text using **BLIP** for captions, **EasyOCR** for visible text, and optional **YOLOv5** for object detection.")
 
158
 
159
  with gr.Row():
160
+ with gr.Column(scale=2):
161
+ file_input = gr.File(
162
+ label="Upload Images",
163
+ file_count="multiple",
164
+ file_types=["image"],
165
+ interactive=True, # Explicitly set interactive
166
+ height=200
167
+ )
168
+ # Moved slider to be more prominent and clearer
169
+ max_len_slider = gr.Slider(
170
+ 50, 500, step=10, value=250, # Increased max length for more flexibility
171
+ label="Maximum Alt Text Length (characters)",
172
+ info="Truncates the generated alt text to this length, preserving word boundaries.",
173
+ interactive=True
174
+ )
175
+
176
+ with gr.Row():
177
+ gen_btn = gr.Button("🚀 Generate Alt Text", variant="primary") # Primary button for main action
178
+ copy_btn = gr.Button("📋 Copy to Clipboard", variant="secondary") # Secondary for less critical
179
+
180
+ with gr.Column(scale=3):
181
+ # Gallery to preview uploaded images
182
+ gallery = gr.Gallery(
183
+ label="Uploaded Image Previews",
184
+ columns=4,
185
+ rows=1,
186
+ object_fit="contain",
187
+ height="auto" # Auto height for better responsiveness
188
+ )
189
+ # Output textbox with a clear ID for JS
190
+ output_text = gr.Textbox(
191
+ label="Generated Alt Text",
192
+ lines=15, # More lines for better readability of multiple alt texts
193
+ interactive=False,
194
+ elem_id="output_text", # Used for JS copy function
195
+ show_copy_button=True # Gradio's built-in copy button
196
+ )
197
+
198
+ # Event Handlers
199
+ # Using 'change' for file_input to update gallery immediately on upload
200
+ file_input.change(fn=lambda files: files, inputs=file_input, outputs=gallery)
201
+
202
+ # Main generation button click
203
+ gen_btn.click(
204
+ fn=generate_alt_text_for_files,
205
+ inputs=[file_input, max_len_slider],
206
+ outputs=[output_text],
207
+ queue=True, # Enable queuing for multiple concurrent requests
208
+ # show_progress=True # Gradio 4.x handles progress bar by default now for queued functions
209
+ )
210
+
211
+ # Custom JS for copy to clipboard if show_copy_button is not enough or for older Gradio versions
212
+ # Note: Gradio 4.x has `show_copy_button=True` for Textbox which is often sufficient.
213
+ # This custom JS is provided as an alternative if needed.
214
+ copy_btn.click(
215
+ None,
216
+ js="""
217
+ () => {
218
+ const outputElement = document.getElementById('output_text');
219
+ if (outputElement) {
220
+ outputElement.select(); // Select the text
221
+ document.execCommand('copy'); // Copy to clipboard
222
+ alert('Alt text copied to clipboard!'); // Provide feedback
223
+ } else {
224
+ alert('Output text element not found.');
225
+ }
226
+ }
227
+ """
228
+ )
229
+
230
+ gr.Markdown(
231
+ """
232
+ ---
233
+ **How it works:**
234
+ - **BLIP:** Generates a descriptive caption of the image's content.
235
+ - **EasyOCR:** Detects and extracts visible text within the image.
236
+ - **YOLOv5 (Optional):** Identifies and lists prominent objects. (Requires `ultralytics` library)
237
+
238
+ This tool is a demonstration and may not be perfect for all images.
239
+ """
240
+ )
241
+
242
  # For local testing
243
  def main():
244
+ app.launch(
245
+ server_name="0.0.0.0",
246
+ server_port=7860,
247
+ share=True,
248
+ debug=True # Enable debug mode for more detailed logs
249
+ )
250
 
251
  if __name__ == "__main__":
252
+ main()