Workpam commited on
Commit
096ff07
·
verified ·
1 Parent(s): 6e1235d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +73 -18
app.py CHANGED
@@ -3,46 +3,55 @@ from PIL import Image
3
  import torchvision.transforms as T
4
  from torchvision.models import detection
5
  from transformers import Blip2Processor, Blip2ForConditionalGeneration
 
6
 
 
7
  device = "cuda" if torch.cuda.is_available() else "cpu"
8
 
9
- # — 1) Object Detector (Faster R‑CNN, COCO labels) —
10
  detector = detection.fasterrcnn_resnet50_fpn(pretrained=True).eval().to(device)
11
  COCO_LABELS = [
12
- '__background__','person','bicycle','car','motorcycle','airplane','bus','train','truck','boat',
13
- # … fill out with all 91 COCO class names …
 
 
 
 
 
 
14
  ]
15
 
16
- # — 2) Caption Model (lighter for speed) —
17
- processor = Blip2Processor.from_pretrained( "Salesforce/blip2-flan-t5-xl", torch_dtype=torch.float16 )
18
- model = Blip2ForConditionalGeneration.from_pretrained( "Salesforce/blip2-flan-t5-xl", torch_dtype=torch.float16 ).to(device)
19
 
20
- # — 3) Pre‑processing pipeline —
21
  transform = T.Compose([
22
  T.Resize((384, 384)),
23
  T.ToTensor(),
24
  ])
25
 
 
26
  def generate_descriptive_text(img_path, max_new_tokens=32):
27
- # a) Load + preprocess
28
  img = Image.open(img_path).convert("RGB")
29
  tensor_img = transform(img).unsqueeze(0).to(device)
30
 
31
- # b) Detect objects (keep only high‑confidence)
32
  with torch.no_grad():
33
  preds = detector(tensor_img)[0]
34
  labels = [
35
- COCO_LABELS[i]
36
- for i, s in zip(preds["labels"].cpu().tolist(), preds["scores"].cpu().tolist())
37
- if s > 0.6
38
  ]
39
  labels = sorted(set(labels))
40
 
41
- # c) Build prompt for grounding
42
- prefix = "Objects: " + (", ".join(labels) or "none detected") + "."
43
  instruction = "Describe this image factually and include key details like colors, materials, function or context."
44
 
45
- # d) Tokenize + generate
46
  inputs = processor(images=img, text=prefix + " " + instruction, return_tensors="pt").to(device)
47
  outputs = model.generate(
48
  **inputs,
@@ -53,6 +62,52 @@ def generate_descriptive_text(img_path, max_new_tokens=32):
53
  )
54
  return processor.decode(outputs[0], skip_special_tokens=True).strip()
55
 
56
- # — Example usage —
57
- if __name__ == "__main__":
58
- print(generate_descriptive_text("my_product.jpg", max_new_tokens=28))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  import torchvision.transforms as T
4
  from torchvision.models import detection
5
  from transformers import Blip2Processor, Blip2ForConditionalGeneration
6
+ import gradio as gr
7
 
8
+ # Device setup
9
  device = "cuda" if torch.cuda.is_available() else "cpu"
10
 
11
+ # 1) Object detector (Faster R-CNN pretrained on COCO)
12
  detector = detection.fasterrcnn_resnet50_fpn(pretrained=True).eval().to(device)
13
  COCO_LABELS = [
14
+ '__background__', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat',
15
+ 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep',
16
+ 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
17
+ 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard',
18
+ 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich',
19
+ 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed',
20
+ 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven',
21
+ 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'
22
  ]
23
 
24
+ # 2) Caption model (BLIP-2 FLAN-T5 XL)
25
+ processor = Blip2Processor.from_pretrained("Salesforce/blip2-flan-t5-xl", torch_dtype=torch.float16)
26
+ model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-flan-t5-xl", torch_dtype=torch.float16).to(device)
27
 
28
+ # Preprocessing transform
29
  transform = T.Compose([
30
  T.Resize((384, 384)),
31
  T.ToTensor(),
32
  ])
33
 
34
+ # Core generation function
35
  def generate_descriptive_text(img_path, max_new_tokens=32):
36
+ # Load and preprocess image
37
  img = Image.open(img_path).convert("RGB")
38
  tensor_img = transform(img).unsqueeze(0).to(device)
39
 
40
+ # Detect objects with high confidence
41
  with torch.no_grad():
42
  preds = detector(tensor_img)[0]
43
  labels = [
44
+ COCO_LABELS[label_idx]
45
+ for label_idx, score in zip(preds["labels"].cpu().tolist(), preds["scores"].cpu().tolist())
46
+ if score > 0.6
47
  ]
48
  labels = sorted(set(labels))
49
 
50
+ # Build prompt grounding
51
+ prefix = "Objects: " + (", ".join(labels) if labels else "none detected") + "."
52
  instruction = "Describe this image factually and include key details like colors, materials, function or context."
53
 
54
+ # Tokenize and generate
55
  inputs = processor(images=img, text=prefix + " " + instruction, return_tensors="pt").to(device)
56
  outputs = model.generate(
57
  **inputs,
 
62
  )
63
  return processor.decode(outputs[0], skip_special_tokens=True).strip()
64
 
65
+ # Wrapper for Gradio UI
66
+ def generate_descriptive_texts(files, max_new_tokens):
67
+ captions = []
68
+ for f in files or []:
69
+ cap = generate_descriptive_text(f.name, max_new_tokens=int(max_new_tokens))
70
+ captions.append(cap.capitalize())
71
+ return "\n\n".join(captions)
72
+
73
+ # --- Gradio Interface ---
74
+ with gr.Blocks(title="Descriptive Text Generator") as demo:
75
+ gr.Markdown("## Descriptive Text Generator\nGrounded, factually accurate, detailed descriptions.")
76
+
77
+ file_input = gr.File(
78
+ label="Upload Images", file_count="multiple", file_types=["image"]
79
+ )
80
+ gallery = gr.Gallery(label="Image Preview", columns=4)
81
+ slider = gr.Slider(
82
+ minimum=16, maximum=64, step=8, value=32, label="Max New Tokens"
83
+ )
84
+ run_btn = gr.Button("Generate")
85
+ clear_btn = gr.Button("Clear All")
86
+ output = gr.Textbox(
87
+ label="Generated Descriptions",
88
+ lines=6,
89
+ show_copy_button=True
90
+ )
91
+
92
+ # Preview uploaded images
93
+ file_input.change(
94
+ fn=lambda files: [Image.open(f.name).convert("RGB") for f in files or []],
95
+ inputs=file_input,
96
+ outputs=gallery
97
+ )
98
+
99
+ # Generate descriptions
100
+ run_btn.click(
101
+ fn=generate_descriptive_texts,
102
+ inputs=[file_input, slider],
103
+ outputs=output,
104
+ queue=True
105
+ )
106
+
107
+ # Clear all inputs and outputs
108
+ clear_btn.click(
109
+ fn=lambda: ([], [], 32, ""),
110
+ outputs=[file_input, gallery, slider, output]
111
+ )
112
+
113
+ demo.launch()