Workpam commited on
Commit
524733a
·
verified ·
1 Parent(s): 404b281

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +36 -25
app.py CHANGED
@@ -1,59 +1,70 @@
1
  import gradio as gr
2
  from transformers import pipeline
3
  from PIL import Image
 
4
 
5
- # --- 1) Pick a lightweight, public model ---
6
- MODEL_NAME = "nlpconnect/vit-gpt2-image-captioning"
 
 
7
 
8
- # --- 2) Instantiate the image-to-text pipeline ---
9
- # We’ll pass all generation arguments inside generate_kwargs
10
  captioner = pipeline(
11
  "image-to-text",
12
- model=MODEL_NAME
 
 
 
13
  )
14
 
15
- # --- 3) Define your generation function ---
16
- def generate_alt_text(files, max_new_tokens):
17
  captions = []
18
  for f in files or []:
19
  img = Image.open(f.name).convert("RGB")
20
- # send everything via generate_kwargs to bypass sanitize checks
21
- res = captioner(
22
  img,
23
  generate_kwargs={
24
- "max_new_tokens": int(max_new_tokens),
25
- "num_beams": 4
 
26
  }
27
  )
28
- text = res[0]["generated_text"].strip().capitalize()
29
  captions.append(text)
30
  return "\n\n".join(captions)
31
 
32
- # --- 4) Build the Gradio UI ---
33
- with gr.Blocks(title="Vit‑GPT2 Image Captioning (Stable)") as demo:
34
- gr.Markdown("# Vit‑GPT2 Image Captioning")
35
- gr.Markdown(f"Using **{MODEL_NAME}**, passing params via `generate_kwargs`")
36
 
37
  with gr.Row():
38
  file_input = gr.File(label="Upload Images", file_count="multiple", file_types=["image"])
39
- slider = gr.Slider(16, 128, value=64, step=16, label="Max New Tokens")
40
- go_btn = gr.Button("Generate Alt‑Text")
41
- clear_btn = gr.Button("Clear All")
42
 
43
  gallery = gr.Gallery(label="Image Preview", columns=4)
44
- output = gr.Textbox(label="Generated Alt‑Text", lines=10, show_copy_button=True)
45
 
46
- # preview images
47
  file_input.change(
48
  fn=lambda files: [Image.open(f.name).convert("RGB") for f in (files or [])],
49
  inputs=file_input,
50
  outputs=gallery
51
  )
52
 
53
- # generate captions
54
- go_btn.click(generate_alt_text, [file_input, slider], output, queue=True)
 
 
 
 
55
 
56
- # clear everything
57
- clear_btn.click(lambda: ([], 64, [], ""), outputs=[file_input, slider, gallery, output])
 
 
58
 
59
  demo.launch()
 
1
  import gradio as gr
2
  from transformers import pipeline
3
  from PIL import Image
4
+ import torch
5
 
6
+ # --- Setup BLIP-2 Base Model Pipeline ---
7
+ # Uses Salesforce/blip2-flan-t5-base for more accurate captions
8
+ MODEL_NAME = "Salesforce/blip2-flan-t5-base"
9
+ DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
10
 
11
+ # Initialize pipeline with trust_remote_code (required for BLIP-2)
 
12
  captioner = pipeline(
13
  "image-to-text",
14
+ model=MODEL_NAME,
15
+ torch_dtype=(torch.float16 if DEVICE=="cuda" else torch.float32),
16
+ device_map="auto",
17
+ trust_remote_code=True
18
  )
19
 
20
+ # --- Caption Generation Function ---
21
+ def generate_alt_text(files, max_tokens):
22
  captions = []
23
  for f in files or []:
24
  img = Image.open(f.name).convert("RGB")
25
+ # Pass generation args via generate_kwargs
26
+ result = captioner(
27
  img,
28
  generate_kwargs={
29
+ "max_new_tokens": int(max_tokens),
30
+ "num_beams": 5,
31
+ "no_repeat_ngram_size": 3
32
  }
33
  )
34
+ text = result[0]["generated_text"].strip().capitalize()
35
  captions.append(text)
36
  return "\n\n".join(captions)
37
 
38
+ # --- Gradio Interface ---
39
+ with gr.Blocks(title="BLIP-2 Base Alt Text Generator") as demo:
40
+ gr.Markdown("# BLIP-2 (FLAN-T5 Base) Alt Text Generator")
41
+ gr.Markdown(f"Model: **{MODEL_NAME}** for improved accuracy")
42
 
43
  with gr.Row():
44
  file_input = gr.File(label="Upload Images", file_count="multiple", file_types=["image"])
45
+ max_tokens = gr.Slider(16, 256, value=64, step=16, label="Max New Tokens")
46
+ generate_btn = gr.Button("Generate Alt Text")
47
+ clear_btn = gr.Button("Clear All")
48
 
49
  gallery = gr.Gallery(label="Image Preview", columns=4)
50
+ output = gr.Textbox(label="Generated Alt Texts", lines=10, show_copy_button=True)
51
 
 
52
  file_input.change(
53
  fn=lambda files: [Image.open(f.name).convert("RGB") for f in (files or [])],
54
  inputs=file_input,
55
  outputs=gallery
56
  )
57
 
58
+ generate_btn.click(
59
+ fn=generate_alt_text,
60
+ inputs=[file_input, max_tokens],
61
+ outputs=output,
62
+ queue=True
63
+ )
64
 
65
+ clear_btn.click(
66
+ fn=lambda: ([], 64, [], ""),
67
+ outputs=[file_input, max_tokens, gallery, output]
68
+ )
69
 
70
  demo.launch()