Workpam commited on
Commit
c749fa1
·
verified ·
1 Parent(s): ff29023

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +51 -24
app.py CHANGED
@@ -6,11 +6,15 @@ import easyocr
6
 
7
  # --- Configuration ---
8
  device = "cuda" if torch.cuda.is_available() else "cpu"
9
- # *** FIX HERE: Change model_name to a valid Hugging Face model identifier ***
10
- model_name = "Salesforce/blip2-flan-t5-xl" # Using the 'xl' version as a common choice
 
 
 
11
 
12
  # Ensure you have sentencepiece installed: pip install sentencepiece
13
- # Load BLIP-2 base model (balanced performance)
 
14
  processor = Blip2Processor.from_pretrained(model_name, use_fast=False)
15
  model = (
16
  Blip2ForConditionalGeneration.from_pretrained(
@@ -19,34 +23,47 @@ model = (
19
  )
20
  .to(device)
21
  )
 
22
 
23
  # Initialize EasyOCR (CPU-friendly)
 
24
  ocr_reader = easyocr.Reader(['en'], gpu=(device == "cuda"))
 
25
 
26
  # --- Core Generation Function ---
27
  def generate_alt_text(files, max_tokens):
28
  captions = []
 
 
 
29
  for f in files:
30
  # Load image
 
 
31
  path = f.name if hasattr(f, 'name') else f
32
  try:
33
  image = Image.open(path).convert("RGB")
34
  except Exception as e:
35
- captions.append(f"Error opening {path}: {e}")
36
  continue
37
 
38
  # OCR extraction
 
39
  try:
40
- ocr_texts = ocr_reader.readtext(path, detail=0)
41
- visible_text = ' '.join(ocr_texts).strip()
42
- except Exception:
43
- visible_text = ''
 
 
 
44
 
45
  # BLIP2 captioning with an enhanced prompt
46
  blip2_prompt = (
47
  "Describe this image in detail for a visually impaired user, "
48
  "focusing on the main subjects, actions, and overall scene. "
49
- "What is happening and what are the key elements?"
 
50
  )
51
 
52
  inputs = processor(
@@ -58,33 +75,39 @@ def generate_alt_text(files, max_tokens):
58
  outputs = model.generate(
59
  **inputs,
60
  max_new_tokens=int(max_tokens),
61
- num_beams=5,
62
  no_repeat_ngram_size=2,
63
- length_penalty=0.8
 
64
  )
 
65
  caption = processor.decode(outputs[0], skip_special_tokens=True).strip()
66
 
67
- # Capitalize the first letter of the caption, ensuring it looks like a sentence.
68
  if caption:
69
  caption = caption[0].upper() + caption[1:]
70
  if not caption.endswith(('.', '!', '?')):
71
- caption += '.' # Ensure the caption ends with punctuation for completeness
 
 
72
 
73
  # Combine with OCR text if any
74
  if visible_text:
75
- if len(caption) > 0 and not caption.endswith('.'):
76
- caption = f"{caption}. Text on image: '{visible_text}'."
77
- else:
78
  caption = f"{caption} Text on image: '{visible_text}'."
 
 
79
 
80
  captions.append(caption)
81
- return captions
82
 
83
  # --- Gradio Interface ---
84
  with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
85
  gr.Markdown("# Balanced High‑End Alt Text Generator")
86
  gr.Markdown(
87
- "This app uses BLIP‑2 Base for high‑quality captions plus EasyOCR for visible text extraction. "
88
  "The aim is to generate descriptive and context-rich alt text, not just simple statements."
89
  )
90
 
@@ -98,8 +121,9 @@ with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
98
  minimum=16,
99
  maximum=128,
100
  step=16,
101
- value=64,
102
- label="Max Tokens for Caption"
 
103
  )
104
  generate_btn = gr.Button("Generate Alt Text")
105
  clear_btn = gr.Button("Clear All")
@@ -108,7 +132,8 @@ with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
108
  output = gr.Textbox(
109
  label="Generated Alt Texts",
110
  lines=10,
111
- show_copy_button=True
 
112
  )
113
 
114
  # Bindings
@@ -117,13 +142,15 @@ with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
117
  fn=generate_alt_text,
118
  inputs=[file_input, max_tokens],
119
  outputs=output,
120
- queue=True
 
121
  )
122
  clear_btn.click(
123
- lambda: ([], 64, []),
124
  inputs=None,
125
  outputs=[file_input, max_tokens, gallery, output]
126
  )
127
 
128
  # Launch
129
- demo.launch(server_name="0.0.0.0", server_port=7860)
 
 
6
 
7
  # --- Configuration ---
8
  device = "cuda" if torch.cuda.is_available() else "cpu"
9
+
10
+ # Using the recommended 'xl' model as 'base' doesn't exist
11
+ # Ensure you have accepted the terms on Hugging Face for this model
12
+ # and logged in via 'huggingface-cli login' if necessary.
13
+ model_name = "Salesforce/blip2-flan-t5-xl"
14
 
15
  # Ensure you have sentencepiece installed: pip install sentencepiece
16
+ # Load BLIP-2 model (balanced performance)
17
+ print(f"Loading BLIP-2 model: {model_name} on device: {device}...")
18
  processor = Blip2Processor.from_pretrained(model_name, use_fast=False)
19
  model = (
20
  Blip2ForConditionalGeneration.from_pretrained(
 
23
  )
24
  .to(device)
25
  )
26
+ print("BLIP-2 model loaded successfully!")
27
 
28
  # Initialize EasyOCR (CPU-friendly)
29
+ print("Initializing EasyOCR...")
30
  ocr_reader = easyocr.Reader(['en'], gpu=(device == "cuda"))
31
+ print("EasyOCR initialized!")
32
 
33
  # --- Core Generation Function ---
34
  def generate_alt_text(files, max_tokens):
35
  captions = []
36
+ if not files:
37
+ return ["Please upload one or more images."]
38
+
39
  for f in files:
40
  # Load image
41
+ # Gradio's gr.File input typically provides a NamedTemporaryFile object,
42
+ # which has a 'name' attribute for the file path.
43
  path = f.name if hasattr(f, 'name') else f
44
  try:
45
  image = Image.open(path).convert("RGB")
46
  except Exception as e:
47
+ captions.append(f"Error opening image {path}: {e}")
48
  continue
49
 
50
  # OCR extraction
51
+ visible_text = ''
52
  try:
53
+ ocr_results = ocr_reader.readtext(path, detail=0)
54
+ if ocr_results:
55
+ visible_text = ' '.join(ocr_results).strip()
56
+ except Exception as e:
57
+ # Catch potential OCR errors but don't stop the main process
58
+ print(f"Warning: EasyOCR failed for {path}: {e}")
59
+ visible_text = '' # Ensure it's empty if OCR fails
60
 
61
  # BLIP2 captioning with an enhanced prompt
62
  blip2_prompt = (
63
  "Describe this image in detail for a visually impaired user, "
64
  "focusing on the main subjects, actions, and overall scene. "
65
+ "What is happening and what are the key elements, including colors, "
66
+ "mood, and context? Be evocative and comprehensive."
67
  )
68
 
69
  inputs = processor(
 
75
  outputs = model.generate(
76
  **inputs,
77
  max_new_tokens=int(max_tokens),
78
+ num_beams=5, # Increased beams for potentially better quality
79
  no_repeat_ngram_size=2,
80
+ length_penalty=0.8, # Added length penalty to encourage more concise yet descriptive output
81
+ early_stopping=True # Stop if all beam hypotheses have met max_new_tokens or EOS token
82
  )
83
+
84
  caption = processor.decode(outputs[0], skip_special_tokens=True).strip()
85
 
86
+ # Post-processing: Capitalize first letter and ensure punctuation
87
  if caption:
88
  caption = caption[0].upper() + caption[1:]
89
  if not caption.endswith(('.', '!', '?')):
90
+ caption += '.'
91
+ else:
92
+ caption = "No clear description generated." # Fallback if BLIP-2 produces empty output
93
 
94
  # Combine with OCR text if any
95
  if visible_text:
96
+ # Integrate OCR text naturally or append it clearly.
97
+ # Avoid redundant punctuation if caption already ends with it.
98
+ if caption.endswith('.'):
99
  caption = f"{caption} Text on image: '{visible_text}'."
100
+ else:
101
+ caption = f"{caption}. Text on image: '{visible_text}'."
102
 
103
  captions.append(caption)
104
+ return "\n\n".join(captions) # Join all captions with double newlines for readability
105
 
106
  # --- Gradio Interface ---
107
  with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
108
  gr.Markdown("# Balanced High‑End Alt Text Generator")
109
  gr.Markdown(
110
+ "This app uses BLIP‑2 Flan‑T5‑XL for high‑quality captions plus EasyOCR for visible text extraction. "
111
  "The aim is to generate descriptive and context-rich alt text, not just simple statements."
112
  )
113
 
 
121
  minimum=16,
122
  maximum=128,
123
  step=16,
124
+ value=64, # Increased default max tokens to allow for more description
125
+ label="Max Tokens for Caption",
126
+ info="Sets the maximum length of the generated image description."
127
  )
128
  generate_btn = gr.Button("Generate Alt Text")
129
  clear_btn = gr.Button("Clear All")
 
132
  output = gr.Textbox(
133
  label="Generated Alt Texts",
134
  lines=10,
135
+ show_copy_button=True,
136
+ interactive=False # Make it read-only
137
  )
138
 
139
  # Bindings
 
142
  fn=generate_alt_text,
143
  inputs=[file_input, max_tokens],
144
  outputs=output,
145
+ queue=True, # Enable queuing for multiple requests
146
+ api_name="generate_alt_text" # Expose this function via API
147
  )
148
  clear_btn.click(
149
+ lambda: ([], 64, [], ""), # Reset file_input, max_tokens, gallery, and output
150
  inputs=None,
151
  outputs=[file_input, max_tokens, gallery, output]
152
  )
153
 
154
  # Launch
155
+ if __name__ == "__main__":
156
+ demo.launch(server_name="0.0.0.0", server_port=7860)