Workpam commited on
Commit
519bafc
·
verified ·
1 Parent(s): 0a09b36

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +80 -121
app.py CHANGED
@@ -1,152 +1,111 @@
1
  import gradio as gr
2
  import torch
3
  from PIL import Image
4
- from torchvision.transforms import Compose, Lambda, Resize, ToTensor, Normalize
5
- from torchvision.transforms.functional import InterpolationMode
6
- from transformers import AutoModel, AutoTokenizer
7
 
8
- # --- Configuration ---
9
- MODEL_NAME = "5CD-AI/Vintern-1B-v3_5"
10
- DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
11
- DTYPE = torch.bfloat16 if DEVICE == "cuda" else torch.float32
12
 
13
- # Image normalization constants
14
- IMAGENET_MEAN = (0.485, 0.456, 0.406)
15
- IMAGENET_STD = (0.229, 0.224, 0.225)
16
-
17
- # --- Preprocessing Helpers ---
18
- def build_transform(input_size):
19
- return Compose([
20
- Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img),
21
- Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
22
- ToTensor(),
23
- Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),
24
- ])
25
-
26
- def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
27
- best_diff = float("inf")
28
- best = (1, 1)
29
- area = width * height
30
- for (i, j) in target_ratios:
31
- targ = i / j
32
- diff = abs(aspect_ratio - targ)
33
- if diff < best_diff or (diff == best_diff and area > 0.5 * image_size * image_size * i * j):
34
- best_diff = diff
35
- best = (i, j)
36
- return best
37
-
38
- def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbnail=False):
39
- w, h = image.size
40
- ar = w / h
41
- # generate all possible splits between min_num and max_num
42
- ratios = sorted(
43
- [(i, j)
44
- for n in range(min_num, max_num+1)
45
- for i in range(1, n+1)
46
- for j in range(1, n+1)
47
- if i*j <= max_num and i*j >= min_num],
48
- key=lambda x: x[0]*x[1]
49
- )
50
- cols, rows = find_closest_aspect_ratio(ar, ratios, w, h, image_size)
51
- target_w, target_h = cols * image_size, rows * image_size
52
- resized = image.resize((target_w, target_h), resample=Image.BICUBIC)
53
-
54
- tiles = []
55
- num_cols = cols
56
- for idx in range(cols * rows):
57
- row_idx = idx // num_cols
58
- col_idx = idx % num_cols
59
- left = col_idx * image_size
60
- upper = row_idx * image_size
61
- right = left + image_size
62
- lower = upper + image_size
63
- tiles.append(resized.crop((left, upper, right, lower)))
64
-
65
- # optionally add a centered thumbnail
66
- if use_thumbnail and len(tiles) != 1:
67
- thumb = image.resize((image_size, image_size), resample=Image.BICUBIC)
68
- tiles.append(thumb)
69
-
70
- return tiles
71
-
72
- def load_image(path, input_size=448, max_num=6):
73
- img = Image.open(path).convert("RGB")
74
- tiles = dynamic_preprocess(img, max_num=max_num, image_size=input_size, use_thumbnail=True)
75
- transform = build_transform(input_size)
76
- pixel_values = torch.stack([transform(t) for t in tiles])
77
- return pixel_values # shape: (num_tiles, 3, input_size, input_size)
78
-
79
- # --- Model & Tokenizer ---
80
- tokenizer = AutoTokenizer.from_pretrained(
81
- MODEL_NAME,
82
- trust_remote_code=True,
83
- use_fast=False
84
- )
85
- model = AutoModel.from_pretrained(
86
- MODEL_NAME,
87
- torch_dtype=DTYPE,
88
- low_cpu_mem_usage=True,
89
  trust_remote_code=True
90
- ).to(DEVICE).eval()
91
-
92
- # Default generation settings
93
- DEFAULT_GEN = {
94
- "max_new_tokens": 128,
95
- "do_sample": False,
96
- "num_beams": 4,
97
- "repetition_penalty": 1.2
98
- }
99
 
100
- # --- Alt-Text Generation ---
101
  def generate_alt_text(files, max_new_tokens):
102
  captions = []
103
  for f in files or []:
104
- # 1) preprocess image into tiles
105
- pixels = load_image(f.name, max_num=6).to(DEVICE, DTYPE)
106
- # 2) prompt the model
107
- prompt = "<image>\nGenerate a concise, descriptive alt text for the image."
108
- # 3) chat-style generation
109
- response, _ = model.chat(
110
- tokenizer,
111
- pixels,
112
- prompt,
113
- {**DEFAULT_GEN, "max_new_tokens": int(max_new_tokens)},
114
- history=None,
115
- return_history=True
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
  )
117
- # 4) collect and clean
118
- cap = response.strip()
119
- if cap:
120
- cap = cap[0].upper() + cap[1:]
121
- captions.append(cap)
122
  return "\n\n".join(captions)
123
 
124
- # --- Gradio UI ---
125
- with gr.Blocks(title="Vintern-1B-v3.5 Alt Text Generator") as demo:
126
- gr.Markdown("# Vintern‑1B‑v3.5 Alt Text Generator")
127
- gr.Markdown("Model fine‑tuned for Vietnamese OCR & document tasks, repurposed for alt‑text.")
 
 
 
128
 
129
  with gr.Row():
130
- file_input = gr.File(label="Upload Images", file_count="multiple", file_types=["image"])
131
- slider = gr.Slider(16, 512, value=128, step=16, label="Max New Tokens")
132
- run_btn = gr.Button("Generate Alt Text")
133
- clear_btn = gr.Button("Clear All")
 
 
 
 
 
 
 
 
 
 
134
 
135
  gallery = gr.Gallery(label="Image Preview", columns=4)
136
- output = gr.Textbox(label="Generated Alt Texts", lines=10, show_copy_button=True)
137
 
 
138
  file_input.change(
139
  fn=lambda files: [Image.open(f.name).convert("RGB") for f in (files or [])],
140
- inputs=file_input, outputs=gallery
 
141
  )
 
 
142
  run_btn.click(
143
  fn=generate_alt_text,
144
  inputs=[file_input, slider],
145
  outputs=output,
146
  queue=True
147
  )
 
 
148
  clear_btn.click(
149
- fn=lambda: ([], 128, [], ""),
150
  outputs=[file_input, slider, gallery, output]
151
  )
152
 
 
1
  import gradio as gr
2
  import torch
3
  from PIL import Image
4
+ import requests
5
+ from transformers import AutoProcessor, AutoModelForCausalLM
 
6
 
7
+ # --- Device & Model Setup ---
8
+ device = "cuda" if torch.cuda.is_available() else "cpu"
9
+ torch_dtype = torch.float16 if device == "cuda" else torch.float32
 
10
 
11
+ # Load model & processor with trust_remote_code
12
+ model = AutoModelForCausalLM.from_pretrained(
13
+ "microsoft/Florence-2-large",
14
+ torch_dtype=torch_dtype,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
15
  trust_remote_code=True
16
+ ).to(device)
17
+ processor = AutoProcessor.from_pretrained(
18
+ "microsoft/Florence-2-large",
19
+ trust_remote_code=True
20
+ )
 
 
 
 
21
 
22
+ # --- Caption Generation Function ---
23
  def generate_alt_text(files, max_new_tokens):
24
  captions = []
25
  for f in files or []:
26
+ # Load image bytes
27
+ image = Image.open(f.name).convert("RGB")
28
+
29
+ # Prepare the Florence prompt (<OD> = object description)
30
+ prompt = "<OD>"
31
+
32
+ # Preprocess
33
+ inputs = processor(
34
+ text=prompt,
35
+ images=image,
36
+ return_tensors="pt"
37
+ ).to(device, torch_dtype)
38
+
39
+ # Generate
40
+ generated_ids = model.generate(
41
+ input_ids=inputs["input_ids"],
42
+ pixel_values=inputs["pixel_values"],
43
+ max_new_tokens=int(max_new_tokens),
44
+ num_beams=3,
45
+ do_sample=False
46
+ )
47
+
48
+ # Decode & post‑process
49
+ generated_text = processor.batch_decode(
50
+ generated_ids,
51
+ skip_special_tokens=False
52
+ )[0]
53
+ parsed = processor.post_process_generation(
54
+ generated_text,
55
+ task=prompt,
56
+ image_size=(image.width, image.height)
57
  )
58
+ caption = parsed.get(prompt, "").strip()
59
+ if caption:
60
+ caption = caption[0].upper() + caption[1:]
61
+ captions.append(caption)
 
62
  return "\n\n".join(captions)
63
 
64
+ # --- Gradio App ---
65
+ with gr.Blocks(title="Florence‑2‑large Alt‑Text Generator") as demo:
66
+ gr.Markdown("# Florence‑2‑large Alt‑Text Generator")
67
+ gr.Markdown(
68
+ "Using **microsoft/Florence-2-large** under the hood. "
69
+ "Drop or upload images and get detailed object descriptions."
70
+ )
71
 
72
  with gr.Row():
73
+ file_input = gr.File(
74
+ label="Upload Images",
75
+ file_count="multiple",
76
+ file_types=["image"]
77
+ )
78
+ slider = gr.Slider(
79
+ minimum=64,
80
+ maximum=2048,
81
+ step=64,
82
+ value=512,
83
+ label="Max New Tokens"
84
+ )
85
+ run_btn = gr.Button("Generate Alt‑Text")
86
+ clear_btn = gr.Button("Clear All")
87
 
88
  gallery = gr.Gallery(label="Image Preview", columns=4)
89
+ output = gr.Textbox(label="Generated Alt‑Text", lines=5, show_copy_button=True)
90
 
91
+ # Preview selected images
92
  file_input.change(
93
  fn=lambda files: [Image.open(f.name).convert("RGB") for f in (files or [])],
94
+ inputs=file_input,
95
+ outputs=gallery
96
  )
97
+
98
+ # Generate alt‑text
99
  run_btn.click(
100
  fn=generate_alt_text,
101
  inputs=[file_input, slider],
102
  outputs=output,
103
  queue=True
104
  )
105
+
106
+ # Clear everything
107
  clear_btn.click(
108
+ fn=lambda: ([], 512, [], ""),
109
  outputs=[file_input, slider, gallery, output]
110
  )
111