Workpam commited on
Commit
593d55c
·
verified ·
1 Parent(s): c749fa1

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +21 -71
app.py CHANGED
@@ -6,15 +6,10 @@ import easyocr
6
 
7
  # --- Configuration ---
8
  device = "cuda" if torch.cuda.is_available() else "cpu"
9
-
10
- # Using the recommended 'xl' model as 'base' doesn't exist
11
- # Ensure you have accepted the terms on Hugging Face for this model
12
- # and logged in via 'huggingface-cli login' if necessary.
13
- model_name = "Salesforce/blip2-flan-t5-xl"
14
 
15
  # Ensure you have sentencepiece installed: pip install sentencepiece
16
- # Load BLIP-2 model (balanced performance)
17
- print(f"Loading BLIP-2 model: {model_name} on device: {device}...")
18
  processor = Blip2Processor.from_pretrained(model_name, use_fast=False)
19
  model = (
20
  Blip2ForConditionalGeneration.from_pretrained(
@@ -23,92 +18,51 @@ model = (
23
  )
24
  .to(device)
25
  )
26
- print("BLIP-2 model loaded successfully!")
27
 
28
  # Initialize EasyOCR (CPU-friendly)
29
- print("Initializing EasyOCR...")
30
  ocr_reader = easyocr.Reader(['en'], gpu=(device == "cuda"))
31
- print("EasyOCR initialized!")
32
 
33
  # --- Core Generation Function ---
34
  def generate_alt_text(files, max_tokens):
35
  captions = []
36
- if not files:
37
- return ["Please upload one or more images."]
38
-
39
  for f in files:
40
  # Load image
41
- # Gradio's gr.File input typically provides a NamedTemporaryFile object,
42
- # which has a 'name' attribute for the file path.
43
  path = f.name if hasattr(f, 'name') else f
44
  try:
45
  image = Image.open(path).convert("RGB")
46
  except Exception as e:
47
- captions.append(f"Error opening image {path}: {e}")
48
  continue
49
-
50
  # OCR extraction
51
- visible_text = ''
52
  try:
53
- ocr_results = ocr_reader.readtext(path, detail=0)
54
- if ocr_results:
55
- visible_text = ' '.join(ocr_results).strip()
56
- except Exception as e:
57
- # Catch potential OCR errors but don't stop the main process
58
- print(f"Warning: EasyOCR failed for {path}: {e}")
59
- visible_text = '' # Ensure it's empty if OCR fails
60
-
61
- # BLIP2 captioning with an enhanced prompt
62
- blip2_prompt = (
63
- "Describe this image in detail for a visually impaired user, "
64
- "focusing on the main subjects, actions, and overall scene. "
65
- "What is happening and what are the key elements, including colors, "
66
- "mood, and context? Be evocative and comprehensive."
67
- )
68
-
69
  inputs = processor(
70
  images=image,
71
- text=blip2_prompt,
72
  return_tensors="pt"
73
  ).to(device)
74
-
75
  outputs = model.generate(
76
  **inputs,
77
  max_new_tokens=int(max_tokens),
78
- num_beams=5, # Increased beams for potentially better quality
79
- no_repeat_ngram_size=2,
80
- length_penalty=0.8, # Added length penalty to encourage more concise yet descriptive output
81
- early_stopping=True # Stop if all beam hypotheses have met max_new_tokens or EOS token
82
  )
83
-
84
- caption = processor.decode(outputs[0], skip_special_tokens=True).strip()
85
-
86
- # Post-processing: Capitalize first letter and ensure punctuation
87
- if caption:
88
- caption = caption[0].upper() + caption[1:]
89
- if not caption.endswith(('.', '!', '?')):
90
- caption += '.'
91
- else:
92
- caption = "No clear description generated." # Fallback if BLIP-2 produces empty output
93
-
94
  # Combine with OCR text if any
95
  if visible_text:
96
- # Integrate OCR text naturally or append it clearly.
97
- # Avoid redundant punctuation if caption already ends with it.
98
- if caption.endswith('.'):
99
- caption = f"{caption} Text on image: '{visible_text}'."
100
- else:
101
- caption = f"{caption}. Text on image: '{visible_text}'."
102
-
103
  captions.append(caption)
104
- return "\n\n".join(captions) # Join all captions with double newlines for readability
105
 
106
  # --- Gradio Interface ---
107
  with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
108
  gr.Markdown("# Balanced High‑End Alt Text Generator")
109
  gr.Markdown(
110
- "This app uses BLIP‑2 Flan‑T5‑XL for high‑quality captions plus EasyOCR for visible text extraction. "
111
- "The aim is to generate descriptive and context-rich alt text, not just simple statements."
112
  )
113
 
114
  with gr.Row():
@@ -121,9 +75,8 @@ with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
121
  minimum=16,
122
  maximum=128,
123
  step=16,
124
- value=64, # Increased default max tokens to allow for more description
125
- label="Max Tokens for Caption",
126
- info="Sets the maximum length of the generated image description."
127
  )
128
  generate_btn = gr.Button("Generate Alt Text")
129
  clear_btn = gr.Button("Clear All")
@@ -132,8 +85,7 @@ with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
132
  output = gr.Textbox(
133
  label="Generated Alt Texts",
134
  lines=10,
135
- show_copy_button=True,
136
- interactive=False # Make it read-only
137
  )
138
 
139
  # Bindings
@@ -142,15 +94,13 @@ with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
142
  fn=generate_alt_text,
143
  inputs=[file_input, max_tokens],
144
  outputs=output,
145
- queue=True, # Enable queuing for multiple requests
146
- api_name="generate_alt_text" # Expose this function via API
147
  )
148
  clear_btn.click(
149
- lambda: ([], 64, [], ""), # Reset file_input, max_tokens, gallery, and output
150
  inputs=None,
151
  outputs=[file_input, max_tokens, gallery, output]
152
  )
153
 
154
  # Launch
155
- if __name__ == "__main__":
156
- demo.launch(server_name="0.0.0.0", server_port=7860)
 
6
 
7
  # --- Configuration ---
8
  device = "cuda" if torch.cuda.is_available() else "cpu"
9
+ model_name = "Salesforce/blip2-flan-t5-base"
 
 
 
 
10
 
11
  # Ensure you have sentencepiece installed: pip install sentencepiece
12
+ # Load BLIP-2 base model (balanced performance)
 
13
  processor = Blip2Processor.from_pretrained(model_name, use_fast=False)
14
  model = (
15
  Blip2ForConditionalGeneration.from_pretrained(
 
18
  )
19
  .to(device)
20
  )
 
21
 
22
  # Initialize EasyOCR (CPU-friendly)
 
23
  ocr_reader = easyocr.Reader(['en'], gpu=(device == "cuda"))
 
24
 
25
  # --- Core Generation Function ---
26
  def generate_alt_text(files, max_tokens):
27
  captions = []
 
 
 
28
  for f in files:
29
  # Load image
 
 
30
  path = f.name if hasattr(f, 'name') else f
31
  try:
32
  image = Image.open(path).convert("RGB")
33
  except Exception as e:
34
+ captions.append(f"Error opening {path}: {e}")
35
  continue
 
36
  # OCR extraction
 
37
  try:
38
+ ocr_texts = ocr_reader.readtext(path, detail=0)
39
+ visible_text = ' '.join(ocr_texts).strip()
40
+ except Exception:
41
+ visible_text = ''
42
+ # BLIP2 captioning
 
 
 
 
 
 
 
 
 
 
 
43
  inputs = processor(
44
  images=image,
45
+ text="Generate a concise and descriptive alt text for this image.",
46
  return_tensors="pt"
47
  ).to(device)
 
48
  outputs = model.generate(
49
  **inputs,
50
  max_new_tokens=int(max_tokens),
51
+ num_beams=3,
52
+ no_repeat_ngram_size=2
 
 
53
  )
54
+ caption = processor.decode(outputs[0], skip_special_tokens=True).strip().capitalize()
 
 
 
 
 
 
 
 
 
 
55
  # Combine with OCR text if any
56
  if visible_text:
57
+ caption = f"{caption} Text on image: '{visible_text}'."
 
 
 
 
 
 
58
  captions.append(caption)
59
+ return captions
60
 
61
  # --- Gradio Interface ---
62
  with gr.Blocks(title="Balanced High‑End Alt Text Generator") as demo:
63
  gr.Markdown("# Balanced High‑End Alt Text Generator")
64
  gr.Markdown(
65
+ "This app uses BLIP‑2 Base for high‑quality captions plus EasyOCR for visible text extraction."
 
66
  )
67
 
68
  with gr.Row():
 
75
  minimum=16,
76
  maximum=128,
77
  step=16,
78
+ value=32,
79
+ label="Max Tokens for Caption"
 
80
  )
81
  generate_btn = gr.Button("Generate Alt Text")
82
  clear_btn = gr.Button("Clear All")
 
85
  output = gr.Textbox(
86
  label="Generated Alt Texts",
87
  lines=10,
88
+ show_copy_button=True
 
89
  )
90
 
91
  # Bindings
 
94
  fn=generate_alt_text,
95
  inputs=[file_input, max_tokens],
96
  outputs=output,
97
+ queue=True
 
98
  )
99
  clear_btn.click(
100
+ lambda: ([], 32, []),
101
  inputs=None,
102
  outputs=[file_input, max_tokens, gallery, output]
103
  )
104
 
105
  # Launch
106
+ demo.launch(server_name="0.0.0.0", server_port=7860)