Workpam commited on
Commit
7e482d8
·
verified ·
1 Parent(s): f6cedd5

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +33 -78
app.py CHANGED
@@ -1,104 +1,59 @@
1
  import gradio as gr
2
- import torch
3
- from transformers import AutoProcessor, AutoModelForCausalLM
4
  from PIL import Image
5
 
6
- # --- Configuration ---
7
- device = "cuda" if torch.cuda.is_available() else "cpu"
8
- torch_dtype = torch.float16 if device == "cuda" else torch.float32
9
- MODEL_NAME = "microsoft/Florence-2-large"
10
 
11
- # Load model and processor with remote code trust
12
- processor = AutoProcessor.from_pretrained(
13
- MODEL_NAME,
14
- trust_remote_code=True
 
15
  )
16
- model = AutoModelForCausalLM.from_pretrained(
17
- MODEL_NAME,
18
- trust_remote_code=True,
19
- torch_dtype=torch_dtype
20
- ).to(device)
21
 
22
- # --- Alt Text Generation ---
23
  def generate_alt_text(files, max_new_tokens):
24
  captions = []
25
  for f in files or []:
26
- # Load image
27
- image = Image.open(f.name).convert("RGB")
28
- # Use caption prompt
29
- prompt = "<CAPTION>"
30
- # Preprocess inputs
31
- inputs = processor(
32
- text=prompt,
33
- images=image,
34
- return_tensors="pt"
35
- ).to(device)
36
- # Generate tokens
37
- generated_ids = model.generate(
38
- input_ids=inputs["input_ids"],
39
- pixel_values=inputs["pixel_values"],
40
- max_new_tokens=int(max_new_tokens),
41
- num_beams=3,
42
- do_sample=False
43
  )
44
- # Decode text
45
- generated_text = processor.batch_decode(
46
- generated_ids,
47
- skip_special_tokens=True
48
- )[0]
49
- # Post-process for caption task
50
- result = processor.post_process_generation(
51
- generated_text,
52
- task=prompt,
53
- image_size=(image.width, image.height)
54
- )
55
- caption = result.get(prompt, "").strip().capitalize()
56
- captions.append(caption)
57
  return "\n\n".join(captions)
58
 
59
- # --- Gradio UI ---
60
- with gr.Blocks(title="Florence-2-large Alt Text Generator") as demo:
61
- gr.Markdown("# Florence-2-large Alt Text Generator")
62
- gr.Markdown("Using microsoft/Florence-2-large with manual model & processor loading.")
63
 
64
  with gr.Row():
65
- file_input = gr.File(
66
- label="Upload Images",
67
- file_count="multiple",
68
- file_types=["image"]
69
- )
70
- max_new_tokens = gr.Slider(
71
- minimum=16,
72
- maximum=512,
73
- step=16,
74
- value=128,
75
- label="Max New Tokens"
76
- )
77
- generate_btn = gr.Button("Generate Alt Text")
78
- clear_btn = gr.Button("Clear All")
79
 
80
  gallery = gr.Gallery(label="Image Preview", columns=4)
81
- output = gr.Textbox(label="Generated Alt Texts", lines=10, show_copy_button=True)
82
 
83
- # Preview images
84
  file_input.change(
85
  fn=lambda files: [Image.open(f.name).convert("RGB") for f in (files or [])],
86
  inputs=file_input,
87
  outputs=gallery
88
  )
89
 
90
- # Generate captions on click
91
- generate_btn.click(
92
- fn=generate_alt_text,
93
- inputs=[file_input, max_new_tokens],
94
- outputs=output,
95
- queue=True
96
- )
97
 
98
- # Clear all fields
99
- clear_btn.click(
100
- fn=lambda: ([], 128, [], ""),
101
- outputs=[file_input, max_new_tokens, gallery, output]
102
- )
103
 
104
  demo.launch()
 
1
  import gradio as gr
2
+ from transformers import pipeline
 
3
  from PIL import Image
4
 
5
+ # --- 1) Pick a lightweight, public model ---
6
+ MODEL_NAME = "nlpconnect/vit-gpt2-image-captioning"
 
 
7
 
8
+ # --- 2) Instantiate the image-to-text pipeline ---
9
+ # We’ll pass all generation arguments inside generate_kwargs
10
+ captioner = pipeline(
11
+ "image-to-text",
12
+ model=MODEL_NAME
13
  )
 
 
 
 
 
14
 
15
+ # --- 3) Define your generation function ---
16
  def generate_alt_text(files, max_new_tokens):
17
  captions = []
18
  for f in files or []:
19
+ img = Image.open(f.name).convert("RGB")
20
+ # send everything via generate_kwargs to bypass sanitize checks
21
+ res = captioner(
22
+ img,
23
+ generate_kwargs={
24
+ "max_new_tokens": int(max_new_tokens),
25
+ "num_beams": 4
26
+ }
 
 
 
 
 
 
 
 
 
27
  )
28
+ text = res[0]["generated_text"].strip().capitalize()
29
+ captions.append(text)
 
 
 
 
 
 
 
 
 
 
 
30
  return "\n\n".join(captions)
31
 
32
+ # --- 4) Build the Gradio UI ---
33
+ with gr.Blocks(title="Vit‑GPT2 Image Captioning (Stable)") as demo:
34
+ gr.Markdown("# Vit‑GPT2 Image Captioning")
35
+ gr.Markdown(f"Using **{MODEL_NAME}**, passing params via `generate_kwargs`")
36
 
37
  with gr.Row():
38
+ file_input = gr.File(label="Upload Images", file_count="multiple", file_types=["image"])
39
+ slider = gr.Slider(16, 128, value=64, step=16, label="Max New Tokens")
40
+ go_btn = gr.Button("Generate Alt‑Text")
41
+ clear_btn = gr.Button("Clear All")
 
 
 
 
 
 
 
 
 
 
42
 
43
  gallery = gr.Gallery(label="Image Preview", columns=4)
44
+ output = gr.Textbox(label="Generated Alt‑Text", lines=10, show_copy_button=True)
45
 
46
+ # preview images
47
  file_input.change(
48
  fn=lambda files: [Image.open(f.name).convert("RGB") for f in (files or [])],
49
  inputs=file_input,
50
  outputs=gallery
51
  )
52
 
53
+ # generate captions
54
+ go_btn.click(generate_alt_text, [file_input, slider], output, queue=True)
 
 
 
 
 
55
 
56
+ # clear everything
57
+ clear_btn.click(lambda: ([], 64, [], ""), outputs=[file_input, slider, gallery, output])
 
 
 
58
 
59
  demo.launch()