Workpam commited on
Commit
0e255ac
·
verified ·
1 Parent(s): d75d03d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +125 -35
app.py CHANGED
@@ -1,63 +1,153 @@
1
  import gradio as gr
2
- from transformers import pipeline
3
- from PIL import Image
4
  import torch
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
 
6
- # --- Setup device & pipeline ---
7
- device = "cuda" if torch.cuda.is_available() else "cpu"
8
- captioner = pipeline(
9
- "image-to-text",
10
- model="Salesforce/blip2-flan-t5-xl",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  trust_remote_code=True,
12
- device_map="auto",
13
- torch_dtype=torch.float16 if device == "cuda" else torch.float32
14
  )
 
 
 
 
 
 
15
 
16
- # --- Caption generation ---
 
 
 
 
 
 
 
 
17
  def generate_alt_text(files, max_new_tokens):
18
  captions = []
19
  for f in files or []:
20
- img = Image.open(f.name).convert("RGB")
21
- out = captioner(
22
- img,
23
- generate_kwargs={
24
- "max_new_tokens": int(max_new_tokens),
25
- "num_beams": 4
26
- }
 
 
 
 
 
27
  )
28
- text = out[0]["generated_text"].strip().capitalize()
29
- captions.append(text)
 
 
 
30
  return "\n\n".join(captions)
31
 
32
  # --- Gradio UI ---
33
- with gr.Blocks(title="BLIP‑2 XL Alt‑Text") as demo:
34
- gr.Markdown("# BLIP‑2 XL Alt‑Text Generator")
35
- gr.Markdown("Model: Salesforce/blip2‑flan‑t5‑xl via `device_map=auto` (requires accelerate)")
36
 
37
  with gr.Row():
38
- files = gr.File(label="Upload Images", file_count="multiple", file_types=["image"])
39
- slider = gr.Slider(16, 256, value=64, step=16, label="Max New Tokens")
40
- run_btn = gr.Button("Generate Alt‑Text")
41
- clear_btn = gr.Button("Clear All")
42
 
43
  gallery = gr.Gallery(label="Image Preview", columns=4)
44
- output = gr.Textbox(label="Generated Alt‑Text", lines=10, show_copy_button=True)
45
 
46
- files.change(
47
- fn=lambda fs: [Image.open(f.name).convert("RGB") for f in (fs or [])],
48
- inputs=files, outputs=gallery
49
  )
50
-
51
  run_btn.click(
52
  fn=generate_alt_text,
53
- inputs=[files, slider],
54
  outputs=output,
55
  queue=True
56
  )
57
-
58
  clear_btn.click(
59
- fn=lambda: ([], 64, [], ""),
60
- outputs=[files, slider, gallery, output]
61
  )
62
 
63
  demo.launch()
 
1
  import gradio as gr
 
 
2
  import torch
3
+ from PIL import Image
4
+ from torchvision.transforms import Compose, Lambda, Resize, ToTensor, Normalize
5
+ from torchvision.transforms.functional import InterpolationMode
6
+ from transformers import AutoModel, AutoTokenizer
7
+
8
+ # --- Configuration ---
9
+ MODEL_NAME = "5CD-AI/Vintern-1B-v3_5"
10
+ DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
11
+ DTYPE = torch.bfloat16 if DEVICE == "cuda" else torch.float32
12
+
13
+ # Image normalization constants
14
+ IMAGENET_MEAN = (0.485, 0.456, 0.406)
15
+ IMAGENET_STD = (0.229, 0.224, 0.225)
16
+
17
+ # --- Preprocessing Helpers ---
18
+ def build_transform(input_size):
19
+ return Compose([
20
+ Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img),
21
+ Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
22
+ ToTensor(),
23
+ Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),
24
+ ])
25
+
26
+ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
27
+ best_diff = float("inf")
28
+ best = (1, 1)
29
+ area = width * height
30
+ for (i, j) in target_ratios:
31
+ targ = i / j
32
+ diff = abs(aspect_ratio - targ)
33
+ if diff < best_diff or (diff == best_diff and area > 0.5 * image_size * image_size * i * j):
34
+ best_diff = diff
35
+ best = (i, j)
36
+ return best
37
+
38
+ def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbnail=False):
39
+ w, h = image.size
40
+ ar = w / h
41
+ # generate all possible splits between min_num and max_num
42
+ ratios = sorted(
43
+ [(i, j)
44
+ for n in range(min_num, max_num+1)
45
+ for i in range(1, n+1)
46
+ for j in range(1, n+1)
47
+ if i*j <= max_num and i*j >= min_num],
48
+ key=lambda x: x[0]*x[1]
49
+ )
50
+ cols, rows = find_closest_aspect_ratio(ar, ratios, w, h, image_size)
51
+ target_w, target_h = cols * image_size, rows * image_size
52
+ resized = image.resize((target_w, target_h), resample=Image.BICUBIC)
53
 
54
+ tiles = []
55
+ num_cols = cols
56
+ for idx in range(cols * rows):
57
+ row_idx = idx // num_cols
58
+ col_idx = idx % num_cols
59
+ left = col_idx * image_size
60
+ upper = row_idx * image_size
61
+ right = left + image_size
62
+ lower = upper + image_size
63
+ tiles.append(resized.crop((left, upper, right, lower)))
64
+
65
+ # optionally add a centered thumbnail
66
+ if use_thumbnail and len(tiles) != 1:
67
+ thumb = image.resize((image_size, image_size), resample=Image.BICUBIC)
68
+ tiles.append(thumb)
69
+
70
+ return tiles
71
+
72
+ def load_image(path, input_size=448, max_num=6):
73
+ img = Image.open(path).convert("RGB")
74
+ tiles = dynamic_preprocess(img, max_num=max_num, image_size=input_size, use_thumbnail=True)
75
+ transform = build_transform(input_size)
76
+ pixel_values = torch.stack([transform(t) for t in tiles])
77
+ return pixel_values # shape: (num_tiles, 3, input_size, input_size)
78
+
79
+ # --- Model & Tokenizer ---
80
+ tokenizer = AutoTokenizer.from_pretrained(
81
+ MODEL_NAME,
82
  trust_remote_code=True,
83
+ use_fast=False
 
84
  )
85
+ model = AutoModel.from_pretrained(
86
+ MODEL_NAME,
87
+ torch_dtype=DTYPE,
88
+ low_cpu_mem_usage=True,
89
+ trust_remote_code=True
90
+ ).to(DEVICE).eval()
91
 
92
+ # Default generation settings
93
+ DEFAULT_GEN = {
94
+ "max_new_tokens": 128,
95
+ "do_sample": False,
96
+ "num_beams": 4,
97
+ "repetition_penalty": 1.2
98
+ }
99
+
100
+ # --- Alt-Text Generation ---
101
  def generate_alt_text(files, max_new_tokens):
102
  captions = []
103
  for f in files or []:
104
+ # 1) preprocess image into tiles
105
+ pixels = load_image(f.name, max_num=6).to(DEVICE, DTYPE)
106
+ # 2) prompt the model
107
+ prompt = "<image>\nGenerate a concise, descriptive alt text for the image."
108
+ # 3) chat-style generation
109
+ response, _ = model.chat(
110
+ tokenizer,
111
+ pixels,
112
+ prompt,
113
+ {**DEFAULT_GEN, "max_new_tokens": int(max_new_tokens)},
114
+ history=None,
115
+ return_history=True
116
  )
117
+ # 4) collect and clean
118
+ cap = response.strip()
119
+ if cap:
120
+ cap = cap[0].upper() + cap[1:]
121
+ captions.append(cap)
122
  return "\n\n".join(captions)
123
 
124
  # --- Gradio UI ---
125
+ with gr.Blocks(title="Vintern-1B-v3.5 Alt Text Generator") as demo:
126
+ gr.Markdown("# Vintern‑1B‑v3.5 Alt Text Generator")
127
+ gr.Markdown("Model fine‑tuned for Vietnamese OCR & document tasks, repurposed for alt‑text.")
128
 
129
  with gr.Row():
130
+ file_input = gr.File(label="Upload Images", file_count="multiple", file_types=["image"])
131
+ slider = gr.Slider(16, 512, value=128, step=16, label="Max New Tokens")
132
+ run_btn = gr.Button("Generate Alt Text")
133
+ clear_btn = gr.Button("Clear All")
134
 
135
  gallery = gr.Gallery(label="Image Preview", columns=4)
136
+ output = gr.Textbox(label="Generated Alt Texts", lines=10, show_copy_button=True)
137
 
138
+ file_input.change(
139
+ fn=lambda files: [Image.open(f.name).convert("RGB") for f in (files or [])],
140
+ inputs=file_input, outputs=gallery
141
  )
 
142
  run_btn.click(
143
  fn=generate_alt_text,
144
+ inputs=[file_input, slider],
145
  outputs=output,
146
  queue=True
147
  )
 
148
  clear_btn.click(
149
+ fn=lambda: ([], 128, [], ""),
150
+ outputs=[file_input, slider, gallery, output]
151
  )
152
 
153
  demo.launch()