Workpam commited on
Commit
b3af92f
·
verified ·
1 Parent(s): 6962353

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +56 -99
app.py CHANGED
@@ -1,102 +1,59 @@
1
- import gradio as gr
2
- from transformers import pipeline
3
  from PIL import Image
4
-
5
- # --- 1) Setup BLIP-2 pipeline ---
6
- captioner = pipeline(
7
- "image-to-text",
8
- model="Salesforce/blip2-flan-t5-xl",
9
- trust_remote_code=True,
10
- device=0 # GPU index, or -1 for CPU
11
- )
12
-
13
- # --- 2) Generation fn ---
14
- # --- 2) Generation fn ---
15
- def generate_alt_text(files, max_new_tokens):
16
- results = []
17
- # Dynamic device selection for robustness
18
- device = 0 if torch.cuda.is_available() else -1
19
- # Re-initialize captioner to ensure device setting is applied,
20
- # or make it a global/cached object if performance is critical for multiple calls
21
- # For a Gradio app, re-initializing inside the fn might be slow if called frequently.
22
- # Ideally, captioner is initialized once globally.
23
-
24
- # Ensure captioner is initialized once globally with proper device and trust_remote_code
25
- # captioner = pipeline(
26
- # "image-to-text",
27
- # model="Salesforce/blip2-flan-t5-xl",
28
- # trust_remote_code=True,
29
- # device=device
30
- # )
31
-
32
- for f in files or []:
33
- img = Image.open(f.name).convert("RGB")
34
- out = captioner(
35
- img,
36
- generate_kwargs={
37
- "max_new_tokens": int(max_new_tokens),
38
- "num_beams": 4, # Start here for speed, can cautiously increase if needed
39
- "no_repeat_ngram_size": 2,
40
- "do_sample": True, # Enable sampling for less generic output
41
- "temperature": 0.8, # Experiment with 0.7 to 1.0
42
- "top_k": 50 # Experiment with 30 to 100, or top_p=0.95
43
- }
44
- )
45
- caption = out[0]["generated_text"].strip().capitalize()
46
- # Optional: Add a period if not present
47
- if not caption.endswith(('.', '!', '?')):
48
- caption += '.'
49
- results.append(caption)
50
- return "\n\n".join(results)
51
-
52
- # --- 3) Gradio UI ---
53
- with gr.Blocks(title="BLIP‑2 Alt‑Text Generator") as demo:
54
- gr.Markdown("## BLIP‑2 FLAN‑T5 XL Alt‑Text Generator")
55
- gr.Markdown("Upload images below and click **Generate** for rich, accurate captions.")
56
-
57
- file_input = gr.File(
58
- label="Upload Images",
59
- file_count="multiple",
60
- file_types=["image"]
61
- )
62
- gallery = gr.Gallery(
63
- label="Image Preview",
64
- columns=4
65
- )
66
- slider = gr.Slider(
67
- minimum=16,
68
- maximum=128,
69
- step=16,
70
- value=64,
71
- label="Max New Tokens"
72
- )
73
- run_btn = gr.Button("Generate")
74
- clear_btn = gr.Button("Clear All")
75
- output = gr.Textbox(
76
- label="Generated Alt‑Text",
77
- lines=6,
78
- show_copy_button=True
79
- )
80
-
81
- # Preview images
82
- file_input.change(
83
- fn=lambda files: [Image.open(f.name).convert("RGB") for f in (files or [])],
84
- inputs=file_input,
85
- outputs=gallery
86
- )
87
-
88
- # Generate captions
89
- run_btn.click(
90
- fn=generate_alt_text,
91
- inputs=[file_input, slider],
92
- outputs=output,
93
- queue=True
94
- )
95
-
96
- # Clear everything
97
- clear_btn.click(
98
- fn=lambda: ([], [], 64, ""),
99
- outputs=[file_input, gallery, slider, output]
100
  )
 
101
 
102
- demo.launch()
 
 
 
1
+ import torch
 
2
  from PIL import Image
3
+ import torchvision.transforms as T
4
+ from torchvision.models import detection
5
+ from transformers import Blip2Processor, Blip2ForConditionalGeneration
6
+
7
+ # — DEVICE SETUP —
8
+ device = "cuda" if torch.cuda.is_available() else "cpu"
9
+
10
+ # — 1) Object Detector (Faster R‑CNN, COCO labels) —
11
+ detector = detection.fasterrcnn_resnet50_fpn(pretrained=True).eval().to(device)
12
+ COCO_LABELS = [
13
+ '__background__','person','bicycle','car','motorcycle','airplane','bus','train','truck','boat',
14
+ # … fill out with all 91 COCO class names …
15
+ ]
16
+
17
+ # — 2) Caption Model (lighter for speed) —
18
+ processor = Blip2Processor.from_pretrained("Salesforce/blip2-flan-t5-base", torch_dtype=torch.float16)
19
+ model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-flan-t5-base", torch_dtype=torch.float16).to(device)
20
+
21
+ # — 3) Pre‑processing pipeline —
22
+ transform = T.Compose([
23
+ T.Resize((384, 384)),
24
+ T.ToTensor(),
25
+ ])
26
+
27
+ def generate_descriptive_text(img_path, max_new_tokens=32):
28
+ # a) Load + preprocess
29
+ img = Image.open(img_path).convert("RGB")
30
+ tensor_img = transform(img).unsqueeze(0).to(device)
31
+
32
+ # b) Detect objects (keep only high‑confidence)
33
+ with torch.no_grad():
34
+ preds = detector(tensor_img)[0]
35
+ labels = [
36
+ COCO_LABELS[i]
37
+ for i, s in zip(preds["labels"].cpu().tolist(), preds["scores"].cpu().tolist())
38
+ if s > 0.6
39
+ ]
40
+ labels = sorted(set(labels))
41
+
42
+ # c) Build prompt for grounding
43
+ prefix = "Objects: " + (", ".join(labels) or "none detected") + "."
44
+ instruction = "Describe this image factually and include key details like colors, materials, function or context."
45
+
46
+ # d) Tokenize + generate
47
+ inputs = processor(images=img, text=prefix + " " + instruction, return_tensors="pt").to(device)
48
+ outputs = model.generate(
49
+ **inputs,
50
+ max_new_tokens=max_new_tokens,
51
+ num_beams=3,
52
+ no_repeat_ngram_size=2,
53
+ early_stopping=True
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
  )
55
+ return processor.decode(outputs[0], skip_special_tokens=True).strip()
56
 
57
+ # — Example usage —
58
+ if __name__ == "__main__":
59
+ print(generate_descriptive_text("my_product.jpg", max_new_tokens=28))