Workpam commited on
Commit
5a5bedd
·
verified ·
1 Parent(s): 5451b34

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +97 -212
app.py CHANGED
@@ -6,257 +6,142 @@ import io
6
  import json
7
  from transformers import BlipProcessor, BlipForConditionalGeneration
8
  import torch
 
 
 
 
 
9
 
10
  # Initialize the BLIP model for image captioning
11
  processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
12
- model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
13
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  def generate_alt_text(image):
15
- """Generate alt text for a single image using BLIP model"""
16
  try:
17
- # Process the image
18
- inputs = processor(image, return_tensors="pt")
19
-
20
- # Generate caption with better parameters to avoid repetition
 
 
 
 
 
21
  with torch.no_grad():
22
  out = model.generate(
23
- **inputs,
24
- max_length=50, # Reduced max length
25
- min_length=5, # Minimum length
26
- num_beams=3, # Reduced beam search
27
  early_stopping=True,
28
- no_repeat_ngram_size=2, # Prevent repetition
29
- do_sample=True, # Add sampling
30
- temperature=0.7, # Control randomness
31
- top_p=0.9, # Nucleus sampling
32
- repetition_penalty=1.2 # Penalize repetition
33
  )
34
-
35
- # Decode the generated text
36
- caption = processor.decode(out[0], skip_special_tokens=True)
37
-
38
- # Clean up repetitive text
39
  caption = clean_repetitive_text(caption)
40
-
41
- # Clean up the caption to make it more suitable as alt text
42
- if caption.lower().startswith("a picture of"):
43
- caption = caption[12:].strip()
44
- elif caption.lower().startswith("an image of"):
45
- caption = caption[11:].strip()
46
- elif caption.lower().startswith("a photo of"):
47
- caption = caption[10:].strip()
48
-
49
- # Remove common prefixes and clean up
50
- caption = caption.strip()
51
-
52
- # Ensure it's not empty and capitalize first letter
53
- if caption and len(caption) > 0:
54
- caption = caption[0].upper() + caption[1:]
55
- else:
56
- caption = "Image content could not be determined"
57
-
58
- # Final check for repetitive patterns
59
  if is_repetitive(caption):
60
  caption = "Image with visual content"
61
-
62
- return caption
63
-
64
- except Exception as e:
65
- return f"Error generating alt text: {str(e)}"
66
 
67
- def clean_repetitive_text(text):
68
- """Clean repetitive text patterns"""
69
- import re
70
-
71
- # Remove patterns like "lan lan lan" or "a a a"
72
- text = re.sub(r'\b(\w+)(\s+\1){2,}\b', r'\1', text)
73
-
74
- # Remove excessive repetition of short words
75
- text = re.sub(r'\b(\w{1,3})\s+(\1\s+){3,}', '', text)
76
-
77
- # Clean up extra spaces
78
- text = ' '.join(text.split())
79
-
80
- return text.strip()
81
-
82
- def is_repetitive(text):
83
- """Check if text is repetitive"""
84
- words = text.split()
85
- if len(words) < 3:
86
- return False
87
-
88
- # Check for repeated words
89
- word_counts = {}
90
- for word in words:
91
- word_counts[word] = word_counts.get(word, 0) + 1
92
-
93
- # If any word appears more than 3 times, it's likely repetitive
94
- for count in word_counts.values():
95
- if count > 3:
96
- return True
97
-
98
- return False
99
 
 
100
  def process_images(files):
101
- """Process multiple uploaded images and generate alt text for each"""
102
- if not files:
103
- return "No images uploaded"
104
-
105
  results = []
106
-
107
- for i, file in enumerate(files):
108
  try:
109
- # Open and process the image
110
- image = Image.open(file.name).convert('RGB')
111
-
112
- # Generate alt text
113
- alt_text = generate_alt_text(image)
114
-
115
- # Create result dictionary
116
- result = {
117
- "image_index": i + 1,
118
- "filename": file.name.split('/')[-1],
119
- "alt_text": alt_text,
120
- "image": image
121
- }
122
- results.append(result)
123
-
124
  except Exception as e:
125
  results.append({
126
- "image_index": i + 1,
127
- "filename": file.name.split('/')[-1] if hasattr(file, 'name') else f"Image {i+1}",
128
- "alt_text": f"Error processing image: {str(e)}",
129
- "image": None
130
  })
131
-
132
  return results
133
 
134
- .filename {
135
- font-weight: bold;
136
- color: #333;
137
- margin-bottom: 5px;
138
- }
139
- </style>
 
 
 
 
140
  """
141
-
142
- for i, result in enumerate(results):
143
- button_id = f"copy-btn-{i}"
144
  html += f"""
145
- <div class="result-container">
146
- <div class="filename">📁 {result['filename']}</div>
147
- <div class="alt-text-container">
148
- <strong>Alt Text:</strong><br>
149
- <span id="alt-text-{i}">{result['alt_text']}</span>
150
- </div>
151
- <button class="copy-button" id="{button_id}" onclick="copyToClipboard(`{result['alt_text']}`, '{button_id}')">
152
- 📋 Copy Alt Text
153
- </button>
154
  </div>
155
  """
156
-
157
  html += "</div>"
158
  return html
159
 
 
160
  def main_process(files):
161
- """Main processing function for the Gradio interface"""
162
  if not files:
163
  return "<p>Please upload at least one image to generate alt text.</p>", []
164
-
165
- # Process all uploaded images
166
  results = process_images(files)
167
-
168
- # Create HTML output
169
- html_output = create_result_html(results)
170
-
171
- # Create gallery of images for display
172
- gallery_images = []
173
- for result in results:
174
- if result['image']:
175
- gallery_images.append(result['image'])
176
-
177
- return html_output, gallery_images
178
 
179
- # Create the Gradio interface
180
- with gr.Blocks(title="Alt Text Generator", theme=gr.themes.Soft()) as demo:
 
181
  gr.Markdown("""
182
  # 🖼️ Alt Text Generator
183
-
184
- Upload multiple images and generate descriptive alt text for each one.
185
- The AI will analyze both visual content and any text present in the images.
186
-
187
- **Features:**
188
- - Support for multiple image uploads
189
- - AI-powered alt text generation
190
- - Copy-to-clipboard functionality for each generated alt text
191
- - Preview of all uploaded images
192
  """)
193
-
194
  with gr.Row():
195
  with gr.Column():
196
- file_input = gr.Files(
197
- label="Upload Images",
198
- file_count="multiple",
199
- file_types=["image"],
200
- height=200
201
- )
202
-
203
- process_btn = gr.Button(
204
- "Generate Alt Text",
205
- variant="primary",
206
- size="lg"
207
- )
208
-
209
  with gr.Column():
210
- image_gallery = gr.Gallery(
211
- label="Uploaded Images",
212
- show_label=True,
213
- elem_id="gallery",
214
- columns=3,
215
- rows=2,
216
- height="auto"
217
- )
218
-
219
- with gr.Row():
220
- output_html = gr.HTML(
221
- label="Generated Alt Text Results",
222
- show_label=True
223
- )
224
-
225
- # Event handlers
226
- process_btn.click(
227
- fn=main_process,
228
- inputs=[file_input],
229
- outputs=[output_html, image_gallery]
230
- )
231
-
232
- # Auto-process when files are uploaded
233
- file_input.change(
234
- fn=main_process,
235
- inputs=[file_input],
236
- outputs=[output_html, image_gallery]
237
- )
238
-
239
- gr.Markdown("""
240
- ### 📝 How to Use:
241
- 1. **Upload Images**: Click "Upload Images" and select one or more image files
242
- 2. **Generate Alt Text**: Click "Generate Alt Text" or wait for automatic processing
243
- 3. **Copy Alt Text**: Use the "Copy Alt Text" button next to each generated description
244
- 4. **Use the Alt Text**: Paste the copied text into your HTML `alt` attributes
245
-
246
- ### 🎯 About Alt Text:
247
- Alt text (alternative text) is crucial for web accessibility. It helps screen readers describe images to visually impaired users and provides context when images fail to load.
248
-
249
- ### 🔧 Technical Details:
250
- - Uses Salesforce's BLIP model for image captioning
251
- - Processes both visual elements and text within images
252
- - Generates human-readable descriptions suitable for alt attributes
253
- """)
254
 
255
- # Launch the app
256
- if __name__ == "__main__":
257
- demo.launch(
258
- server_name="0.0.0.0",
259
- server_port=7860,
260
- share=True,
261
- debug=True
262
- )
 
6
  import json
7
  from transformers import BlipProcessor, BlipForConditionalGeneration
8
  import torch
9
+ import easyocr
10
+
11
+ # Initialize OCR reader (English)
12
+ device = "cuda" if torch.cuda.is_available() else "cpu"
13
+ ocr_reader = easyocr.Reader(['en'], gpu=(device=='cuda'))
14
 
15
  # Initialize the BLIP model for image captioning
16
  processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
17
+ model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base").to(device)
18
 
19
+ # Helper functions for cleaning captions
20
+ def clean_repetitive_text(text):
21
+ import re
22
+ text = re.sub(r'\b(\w+)(\s+\1){2,}\b', r'\1', text)
23
+ text = re.sub(r'\b(\w{1,3})\s+(\1\s+){3,}', '', text)
24
+ return ' '.join(text.split()).strip()
25
+
26
+ def is_repetitive(text):
27
+ words = text.split()
28
+ if len(words) < 3:
29
+ return False
30
+ from collections import Counter
31
+ return any(count > 3 for count in Counter(words).values())
32
+
33
+ # Generate alt text for a single image
34
  def generate_alt_text(image):
 
35
  try:
36
+ # OCR to extract visible text
37
+ buf = io.BytesIO()
38
+ image.save(buf, format='PNG')
39
+ buf.seek(0)
40
+ ocr_results = ocr_reader.readtext(buf, detail=0)
41
+ visible_text = ' '.join(ocr_results).strip() or '<no text detected>'
42
+
43
+ # Generate caption
44
+ inputs = processor(images=image, return_tensors="pt").to(device)
45
  with torch.no_grad():
46
  out = model.generate(
47
+ **inputs,
48
+ max_length=50,
49
+ min_length=5,
50
+ num_beams=3,
51
  early_stopping=True,
52
+ no_repeat_ngram_size=2,
53
+ temperature=0.7,
54
+ top_p=0.9,
55
+ repetition_penalty=1.2
 
56
  )
57
+ caption = processor.decode(out[0], skip_special_tokens=True).strip()
58
+
59
+ # Clean caption
 
 
60
  caption = clean_repetitive_text(caption)
61
+ for prefix in ["a picture of", "an image of", "a photo of"]:
62
+ if caption.lower().startswith(prefix):
63
+ caption = caption[len(prefix):].strip()
64
+ caption = caption.capitalize() if caption else "Image content could not be determined"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
  if is_repetitive(caption):
66
  caption = "Image with visual content"
 
 
 
 
 
67
 
68
+ # Combine with OCR text
69
+ return f"{caption} (Text on image: {visible_text})"
70
+ except Exception as e:
71
+ return f"Error generating alt text: {e}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
72
 
73
+ # Process multiple files
74
  def process_images(files):
 
 
 
 
75
  results = []
76
+ for i, f in enumerate(files):
77
+ path = f.name if hasattr(f, 'name') else None
78
  try:
79
+ img = Image.open(path).convert('RGB') if path else Image.open(f).convert('RGB')
80
+ alt = generate_alt_text(img)
81
+ results.append({
82
+ 'filename': path.split('/')[-1] if path else f'Image {i+1}',
83
+ 'alt_text': alt,
84
+ 'image': img
85
+ })
 
 
 
 
 
 
 
 
86
  except Exception as e:
87
  results.append({
88
+ 'filename': path.split('/')[-1] if path else f'Image {i+1}',
89
+ 'alt_text': f"Error processing image: {e}",
90
+ 'image': None
 
91
  })
 
92
  return results
93
 
94
+ # Create HTML to display results with copy buttons
95
+ def create_result_html(results):
96
+ html = """
97
+ <style>
98
+ .result-container { margin-bottom: 1rem; }
99
+ .result-container .filename { font-weight: bold; color: #333; margin-bottom: 5px; }
100
+ .result-container .alt-text-container { margin-bottom: 5px; }
101
+ .copy-button { padding: 0.3em 0.6em; margin-top: 5px; }
102
+ </style>
103
+ <div>
104
  """
105
+ for i, r in enumerate(results):
 
 
106
  html += f"""
107
+ <div class=\"result-container\">
108
+ <div class=\"filename\">📁 {r['filename']}</div>
109
+ <div class=\"alt-text-container\">
110
+ <strong>Alt Text:</strong><br>
111
+ <span id=\"alt-text-{i}\">{r['alt_text']}</span>
112
+ </div>
113
+ <button class=\"copy-button\" onclick=\"navigator.clipboard.writeText(document.getElementById('alt-text-{i}').innerText)\">
114
+ 📋 Copy Alt Text
115
+ </button>
116
  </div>
117
  """
 
118
  html += "</div>"
119
  return html
120
 
121
+ # Main processing function for Gradio
122
  def main_process(files):
 
123
  if not files:
124
  return "<p>Please upload at least one image to generate alt text.</p>", []
 
 
125
  results = process_images(files)
126
+ html = create_result_html(results)
127
+ gallery = [r['image'] for r in results if r['image']]
128
+ return html, gallery
 
 
 
 
 
 
 
 
129
 
130
+ # Build the Gradio app
131
+ demo = gr.Blocks()
132
+ with demo:
133
  gr.Markdown("""
134
  # 🖼️ Alt Text Generator
135
+ Upload multiple images and generate descriptive alt text for each one. The AI analyzes visual content and any text present.
 
 
 
 
 
 
 
 
136
  """)
 
137
  with gr.Row():
138
  with gr.Column():
139
+ file_input = gr.Files(label="Upload Images", file_count="multiple", file_types=["image"], height=200)
140
+ process_btn = gr.Button("Generate Alt Text", variant="primary", size="lg")
 
 
 
 
 
 
 
 
 
 
 
141
  with gr.Column():
142
+ gallery = gr.Gallery(label="Uploaded Images", show_label=True, columns=3)
143
+ output_html = gr.HTML(label="Generated Alt Text Results")
144
+ process_btn.click(fn=main_process, inputs=file_input, outputs=[output_html, gallery])
145
+ file_input.change(fn=main_process, inputs=file_input, outputs=[output_html, gallery])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
146
 
147
+ demo.launch(server_name="0.0.0.0", server_port=7860, share=True, debug=True)