Update app.py
Browse files
app.py
CHANGED
|
@@ -24,13 +24,16 @@ class Config:
|
|
| 24 |
@lru_cache(maxsize=1) # Cache the BLIP model and processor
|
| 25 |
def load_blip_model():
|
| 26 |
"""Loads the BLIP model and processor."""
|
|
|
|
| 27 |
processor = BlipProcessor.from_pretrained(Config.BLIP_MODEL_NAME)
|
| 28 |
model = BlipForConditionalGeneration.from_pretrained(Config.BLIP_MODEL_NAME).to(Config.DEVICE)
|
|
|
|
| 29 |
return processor, model
|
| 30 |
|
| 31 |
@lru_cache(maxsize=1) # Cache the EasyOCR reader
|
| 32 |
def load_ocr_reader():
|
| 33 |
"""Initializes EasyOCR reader."""
|
|
|
|
| 34 |
return easyocr.Reader(Config.EASYOCR_LANGUAGES, gpu=torch.cuda.is_available())
|
| 35 |
|
| 36 |
@lru_cache(maxsize=1) # Cache the YOLO model
|
|
@@ -38,12 +41,14 @@ def load_yolo_model():
|
|
| 38 |
"""Optionally loads YOLOv5 for object detection."""
|
| 39 |
try:
|
| 40 |
from ultralytics import YOLO
|
|
|
|
| 41 |
yolo_model = YOLO(Config.YOLO_MODEL_NAME)
|
| 42 |
if torch.cuda.is_available():
|
| 43 |
yolo_model.to(Config.DEVICE)
|
|
|
|
| 44 |
return yolo_model
|
| 45 |
except ImportError:
|
| 46 |
-
print("YOLOv5
|
| 47 |
return None
|
| 48 |
except Exception as e:
|
| 49 |
print(f"Error loading YOLO model: {e}. Object detection will be disabled.")
|
|
@@ -62,12 +67,14 @@ def process_single_image(image_path, max_alt_text_len):
|
|
| 62 |
Handles image loading, OCR, object detection, and BLIP captioning.
|
| 63 |
"""
|
| 64 |
filename = os.path.basename(image_path)
|
|
|
|
| 65 |
|
| 66 |
try:
|
| 67 |
# Open and convert image once
|
| 68 |
image = Image.open(image_path).convert('RGB')
|
| 69 |
image.verify() # Verify image integrity
|
| 70 |
except Exception as e:
|
|
|
|
| 71 |
return filename, f"<Error opening or verifying file: {e}>"
|
| 72 |
|
| 73 |
# Initialize results
|
|
@@ -75,24 +82,27 @@ def process_single_image(image_path, max_alt_text_len):
|
|
| 75 |
object_labels = ""
|
| 76 |
ocr_text = ""
|
| 77 |
|
| 78 |
-
# BLIP captioning
|
| 79 |
try:
|
| 80 |
inputs = blip_processor(images=image, return_tensors="pt").to(Config.DEVICE)
|
| 81 |
output_ids = blip_model.generate(
|
| 82 |
**inputs,
|
| 83 |
num_beams=Config.NUM_BLIP_BEAMS,
|
| 84 |
max_length=Config.MAX_CAPTION_LENGTH,
|
| 85 |
-
min_length=Config.
|
| 86 |
no_repeat_ngram_size=Config.NO_REPEAT_NGRAM_SIZE
|
| 87 |
)
|
| 88 |
blip_caption = blip_processor.decode(output_ids[0], skip_special_tokens=True).strip()
|
| 89 |
except Exception as e:
|
| 90 |
print(f"Error generating BLIP caption for {filename}: {e}")
|
| 91 |
-
|
|
|
|
| 92 |
|
| 93 |
# OCR
|
| 94 |
try:
|
| 95 |
-
|
|
|
|
|
|
|
| 96 |
ocr_text = ' '.join(texts).strip()
|
| 97 |
except Exception as e:
|
| 98 |
print(f"Error performing OCR for {filename}: {e}")
|
|
@@ -101,18 +111,30 @@ def process_single_image(image_path, max_alt_text_len):
|
|
| 101 |
# Object detection
|
| 102 |
if use_yolo:
|
| 103 |
try:
|
|
|
|
| 104 |
results = yolo_model(image_path)
|
| 105 |
-
#
|
| 106 |
-
#
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
except Exception as e:
|
| 111 |
print(f"Error performing object detection for {filename}: {e}")
|
| 112 |
object_labels = '' # Keep empty if object detection fails
|
| 113 |
|
| 114 |
# Assemble and truncate alt text
|
| 115 |
-
parts = [
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
if object_labels:
|
| 117 |
parts.append(f"Objects: {object_labels}")
|
| 118 |
if ocr_text:
|
|
@@ -122,10 +144,17 @@ def process_single_image(image_path, max_alt_text_len):
|
|
| 122 |
|
| 123 |
# Truncate to max_alt_text_len, respecting word boundaries
|
| 124 |
if len(alt_text) > max_alt_text_len:
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
|
|
|
|
| 129 |
return filename, alt_text
|
| 130 |
|
| 131 |
def generate_alt_text_for_files(file_objs, max_alt_text_len):
|
|
@@ -133,9 +162,11 @@ def generate_alt_text_for_files(file_objs, max_alt_text_len):
|
|
| 133 |
Generates alt text for multiple image files in parallel.
|
| 134 |
"""
|
| 135 |
if not file_objs:
|
| 136 |
-
return ""
|
| 137 |
|
|
|
|
| 138 |
image_paths = [getattr(f, 'name', f) for f in file_objs]
|
|
|
|
| 139 |
|
| 140 |
# Use ThreadPoolExecutor for concurrent processing
|
| 141 |
# The number of workers can be tuned, but Config.CPU_CORES is a good starting point.
|
|
@@ -145,7 +176,8 @@ def generate_alt_text_for_files(file_objs, max_alt_text_len):
|
|
| 145 |
|
| 146 |
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
| 147 |
# Map the single image processing function to each file path
|
| 148 |
-
|
|
|
|
| 149 |
|
| 150 |
# Format the results for display
|
| 151 |
lines = [f"{fn}: {txt}" for fn, txt in results]
|
|
@@ -154,7 +186,11 @@ def generate_alt_text_for_files(file_objs, max_alt_text_len):
|
|
| 154 |
# 4. Gradio Interface (UI/UX Improvements)
|
| 155 |
with gr.Blocks(title="AI-Powered Alt Text Generator", theme=gr.themes.Soft()) as app: # Added a theme
|
| 156 |
gr.Markdown("# ✨ AI-Powered Alt Text Generator ✨")
|
| 157 |
-
gr.Markdown(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
|
| 159 |
with gr.Row():
|
| 160 |
with gr.Column(scale=2):
|
|
@@ -192,7 +228,7 @@ with gr.Blocks(title="AI-Powered Alt Text Generator", theme=gr.themes.Soft()) as
|
|
| 192 |
lines=15, # More lines for better readability of multiple alt texts
|
| 193 |
interactive=False,
|
| 194 |
elem_id="output_text", # Used for JS copy function
|
| 195 |
-
show_copy_button=True # Gradio's built-in copy button
|
| 196 |
)
|
| 197 |
|
| 198 |
# Event Handlers
|
|
@@ -210,16 +246,25 @@ with gr.Blocks(title="AI-Powered Alt Text Generator", theme=gr.themes.Soft()) as
|
|
| 210 |
|
| 211 |
# Custom JS for copy to clipboard if show_copy_button is not enough or for older Gradio versions
|
| 212 |
# Note: Gradio 4.x has `show_copy_button=True` for Textbox which is often sufficient.
|
| 213 |
-
# This custom JS is provided as an alternative if needed.
|
| 214 |
copy_btn.click(
|
| 215 |
-
None,
|
|
|
|
|
|
|
| 216 |
js="""
|
| 217 |
() => {
|
| 218 |
const outputElement = document.getElementById('output_text');
|
| 219 |
if (outputElement) {
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
} else {
|
| 224 |
alert('Output text element not found.');
|
| 225 |
}
|
|
@@ -235,17 +280,20 @@ with gr.Blocks(title="AI-Powered Alt Text Generator", theme=gr.themes.Soft()) as
|
|
| 235 |
- **EasyOCR:** Detects and extracts visible text within the image.
|
| 236 |
- **YOLOv5 (Optional):** Identifies and lists prominent objects. (Requires `ultralytics` library)
|
| 237 |
|
| 238 |
-
This tool is a demonstration and may not be perfect for all images.
|
|
|
|
| 239 |
"""
|
| 240 |
)
|
| 241 |
|
| 242 |
# For local testing
|
| 243 |
def main():
|
|
|
|
|
|
|
| 244 |
app.launch(
|
| 245 |
-
server_name="0.0.0.0",
|
| 246 |
-
server_port=7860,
|
| 247 |
-
share=True,
|
| 248 |
-
debug=True
|
| 249 |
)
|
| 250 |
|
| 251 |
if __name__ == "__main__":
|
|
|
|
| 24 |
@lru_cache(maxsize=1) # Cache the BLIP model and processor
|
| 25 |
def load_blip_model():
|
| 26 |
"""Loads the BLIP model and processor."""
|
| 27 |
+
print(f"Loading BLIP model '{Config.BLIP_MODEL_NAME}' to {Config.DEVICE}...")
|
| 28 |
processor = BlipProcessor.from_pretrained(Config.BLIP_MODEL_NAME)
|
| 29 |
model = BlipForConditionalGeneration.from_pretrained(Config.BLIP_MODEL_NAME).to(Config.DEVICE)
|
| 30 |
+
print("BLIP model loaded.")
|
| 31 |
return processor, model
|
| 32 |
|
| 33 |
@lru_cache(maxsize=1) # Cache the EasyOCR reader
|
| 34 |
def load_ocr_reader():
|
| 35 |
"""Initializes EasyOCR reader."""
|
| 36 |
+
print(f"Initializing EasyOCR reader for languages {Config.EASYOCR_LANGUAGES} (GPU: {torch.cuda.is_available()})...")
|
| 37 |
return easyocr.Reader(Config.EASYOCR_LANGUAGES, gpu=torch.cuda.is_available())
|
| 38 |
|
| 39 |
@lru_cache(maxsize=1) # Cache the YOLO model
|
|
|
|
| 41 |
"""Optionally loads YOLOv5 for object detection."""
|
| 42 |
try:
|
| 43 |
from ultralytics import YOLO
|
| 44 |
+
print(f"Loading YOLO model '{Config.YOLO_MODEL_NAME}'...")
|
| 45 |
yolo_model = YOLO(Config.YOLO_MODEL_NAME)
|
| 46 |
if torch.cuda.is_available():
|
| 47 |
yolo_model.to(Config.DEVICE)
|
| 48 |
+
print("YOLO model loaded.")
|
| 49 |
return yolo_model
|
| 50 |
except ImportError:
|
| 51 |
+
print("YOLOv5 (ultralytics library) not installed. Object detection will be disabled.")
|
| 52 |
return None
|
| 53 |
except Exception as e:
|
| 54 |
print(f"Error loading YOLO model: {e}. Object detection will be disabled.")
|
|
|
|
| 67 |
Handles image loading, OCR, object detection, and BLIP captioning.
|
| 68 |
"""
|
| 69 |
filename = os.path.basename(image_path)
|
| 70 |
+
print(f"Processing image: {filename}")
|
| 71 |
|
| 72 |
try:
|
| 73 |
# Open and convert image once
|
| 74 |
image = Image.open(image_path).convert('RGB')
|
| 75 |
image.verify() # Verify image integrity
|
| 76 |
except Exception as e:
|
| 77 |
+
print(f"Error opening or verifying file {filename}: {e}")
|
| 78 |
return filename, f"<Error opening or verifying file: {e}>"
|
| 79 |
|
| 80 |
# Initialize results
|
|
|
|
| 82 |
object_labels = ""
|
| 83 |
ocr_text = ""
|
| 84 |
|
| 85 |
+
# BLIP captioning
|
| 86 |
try:
|
| 87 |
inputs = blip_processor(images=image, return_tensors="pt").to(Config.DEVICE)
|
| 88 |
output_ids = blip_model.generate(
|
| 89 |
**inputs,
|
| 90 |
num_beams=Config.NUM_BLIP_BEAMS,
|
| 91 |
max_length=Config.MAX_CAPTION_LENGTH,
|
| 92 |
+
min_length=Config.MIN_CAPPTION_LENGTH,
|
| 93 |
no_repeat_ngram_size=Config.NO_REPEAT_NGRAM_SIZE
|
| 94 |
)
|
| 95 |
blip_caption = blip_processor.decode(output_ids[0], skip_special_tokens=True).strip()
|
| 96 |
except Exception as e:
|
| 97 |
print(f"Error generating BLIP caption for {filename}: {e}")
|
| 98 |
+
# If BLIP fails, we still want to try other components, so don't return early
|
| 99 |
+
blip_caption = f"BLIP error: {e}" # Indicate BLIP error within caption if others succeed
|
| 100 |
|
| 101 |
# OCR
|
| 102 |
try:
|
| 103 |
+
# EasyOCR can take file path directly, or a PIL Image object
|
| 104 |
+
# Using path is often more memory efficient for EasyOCR
|
| 105 |
+
texts = ocr_reader.readtext(image_path, detail=0)
|
| 106 |
ocr_text = ' '.join(texts).strip()
|
| 107 |
except Exception as e:
|
| 108 |
print(f"Error performing OCR for {filename}: {e}")
|
|
|
|
| 111 |
# Object detection
|
| 112 |
if use_yolo:
|
| 113 |
try:
|
| 114 |
+
# YOLO can also take file path directly
|
| 115 |
results = yolo_model(image_path)
|
| 116 |
+
# Access results based on ultralytics YOLO's structure
|
| 117 |
+
# For a single image, results will be a list with one item
|
| 118 |
+
if results and len(results) > 0:
|
| 119 |
+
# The .pandas() method is often convenient for DataFrame access
|
| 120 |
+
# For recent ultralytics versions, results[0].boxes.data.cpu().numpy()
|
| 121 |
+
# or results[0].names might be needed for direct access.
|
| 122 |
+
# Assuming .pandas() for compatibility with original code's approach.
|
| 123 |
+
df = results[0].pandas().xyxy[0] # Assuming results[0] is the detection object
|
| 124 |
+
labels_list = df['name'].unique().tolist()[:5] # Limit to top 5 unique objects
|
| 125 |
+
object_labels = ', '.join(labels_list)
|
| 126 |
+
else:
|
| 127 |
+
object_labels = ''
|
| 128 |
except Exception as e:
|
| 129 |
print(f"Error performing object detection for {filename}: {e}")
|
| 130 |
object_labels = '' # Keep empty if object detection fails
|
| 131 |
|
| 132 |
# Assemble and truncate alt text
|
| 133 |
+
parts = []
|
| 134 |
+
# Always include BLIP caption first if available, even if it's an error message
|
| 135 |
+
if blip_caption:
|
| 136 |
+
parts.append(blip_caption)
|
| 137 |
+
|
| 138 |
if object_labels:
|
| 139 |
parts.append(f"Objects: {object_labels}")
|
| 140 |
if ocr_text:
|
|
|
|
| 144 |
|
| 145 |
# Truncate to max_alt_text_len, respecting word boundaries
|
| 146 |
if len(alt_text) > max_alt_text_len:
|
| 147 |
+
# Find the last space before truncation point
|
| 148 |
+
truncated_text = alt_text[:max_alt_text_len]
|
| 149 |
+
last_space_index = truncated_text.rfind(' ')
|
| 150 |
+
|
| 151 |
+
if last_space_index != -1 and last_space_index > max_alt_text_len * 0.8: # Ensure meaningful truncation
|
| 152 |
+
alt_text = truncated_text[:last_space_index] + '…'
|
| 153 |
+
else:
|
| 154 |
+
# If no space or space is too far back, just cut and add ellipsis
|
| 155 |
+
alt_text = truncated_text + '…'
|
| 156 |
|
| 157 |
+
print(f"Finished processing {filename}. Alt text length: {len(alt_text)}")
|
| 158 |
return filename, alt_text
|
| 159 |
|
| 160 |
def generate_alt_text_for_files(file_objs, max_alt_text_len):
|
|
|
|
| 162 |
Generates alt text for multiple image files in parallel.
|
| 163 |
"""
|
| 164 |
if not file_objs:
|
| 165 |
+
return "No files uploaded."
|
| 166 |
|
| 167 |
+
# file_obj.name gives the path for temp files uploaded to Gradio
|
| 168 |
image_paths = [getattr(f, 'name', f) for f in file_objs]
|
| 169 |
+
print(f"Generating alt text for {len(image_paths)} files.")
|
| 170 |
|
| 171 |
# Use ThreadPoolExecutor for concurrent processing
|
| 172 |
# The number of workers can be tuned, but Config.CPU_CORES is a good starting point.
|
|
|
|
| 176 |
|
| 177 |
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
| 178 |
# Map the single image processing function to each file path
|
| 179 |
+
# executor.map returns results in the order the tasks were submitted
|
| 180 |
+
results = list(executor.map(lambda p: process_single_image(p, max_alt_text_len), image_paths))
|
| 181 |
|
| 182 |
# Format the results for display
|
| 183 |
lines = [f"{fn}: {txt}" for fn, txt in results]
|
|
|
|
| 186 |
# 4. Gradio Interface (UI/UX Improvements)
|
| 187 |
with gr.Blocks(title="AI-Powered Alt Text Generator", theme=gr.themes.Soft()) as app: # Added a theme
|
| 188 |
gr.Markdown("# ✨ AI-Powered Alt Text Generator ✨")
|
| 189 |
+
gr.Markdown(
|
| 190 |
+
"Upload images to generate rich alt text using **BLIP** for captions, "
|
| 191 |
+
"**EasyOCR** for visible text, and optional **YOLOv5** for object detection. "
|
| 192 |
+
"Models are loaded once for faster subsequent processing."
|
| 193 |
+
)
|
| 194 |
|
| 195 |
with gr.Row():
|
| 196 |
with gr.Column(scale=2):
|
|
|
|
| 228 |
lines=15, # More lines for better readability of multiple alt texts
|
| 229 |
interactive=False,
|
| 230 |
elem_id="output_text", # Used for JS copy function
|
| 231 |
+
show_copy_button=True # Gradio's built-in copy button (Gradio 4.x+)
|
| 232 |
)
|
| 233 |
|
| 234 |
# Event Handlers
|
|
|
|
| 246 |
|
| 247 |
# Custom JS for copy to clipboard if show_copy_button is not enough or for older Gradio versions
|
| 248 |
# Note: Gradio 4.x has `show_copy_button=True` for Textbox which is often sufficient.
|
| 249 |
+
# This custom JS is provided as an alternative if needed, but the built-in one is generally better.
|
| 250 |
copy_btn.click(
|
| 251 |
+
None, # No Python function for this JS-only action
|
| 252 |
+
inputs=None, # No inputs needed from Python
|
| 253 |
+
outputs=None, # No outputs to update in Python
|
| 254 |
js="""
|
| 255 |
() => {
|
| 256 |
const outputElement = document.getElementById('output_text');
|
| 257 |
if (outputElement) {
|
| 258 |
+
// Use the modern clipboard API if available, fallback to execCommand
|
| 259 |
+
if (navigator.clipboard && navigator.clipboard.writeText) {
|
| 260 |
+
navigator.clipboard.writeText(outputElement.value)
|
| 261 |
+
.then(() => alert('Alt text copied to clipboard!'))
|
| 262 |
+
.catch(err => console.error('Failed to copy text: ', err));
|
| 263 |
+
} else {
|
| 264 |
+
outputElement.select(); // Select the text
|
| 265 |
+
document.execCommand('copy'); // Copy to clipboard (deprecated but still works in many browsers)
|
| 266 |
+
alert('Alt text copied to clipboard!'); // Provide feedback
|
| 267 |
+
}
|
| 268 |
} else {
|
| 269 |
alert('Output text element not found.');
|
| 270 |
}
|
|
|
|
| 280 |
- **EasyOCR:** Detects and extracts visible text within the image.
|
| 281 |
- **YOLOv5 (Optional):** Identifies and lists prominent objects. (Requires `ultralytics` library)
|
| 282 |
|
| 283 |
+
This tool is a demonstration and may not be perfect for all images.
|
| 284 |
+
For optimal performance, ensure you have a GPU (CUDA) enabled.
|
| 285 |
"""
|
| 286 |
)
|
| 287 |
|
| 288 |
# For local testing
|
| 289 |
def main():
|
| 290 |
+
print(f"\nStarting Gradio app on device: {Config.DEVICE}")
|
| 291 |
+
print(f"YOLO enabled: {use_yolo}\n")
|
| 292 |
app.launch(
|
| 293 |
+
server_name="0.0.0.0", # Makes the app accessible from other devices on the network
|
| 294 |
+
server_port=7860, # Specific port
|
| 295 |
+
share=True, # Creates a public Gradio share link (temporary)
|
| 296 |
+
debug=True # Enable debug mode for more detailed logs in console
|
| 297 |
)
|
| 298 |
|
| 299 |
if __name__ == "__main__":
|