shinydatatech commited on
Commit
53d419e
Β·
verified Β·
1 Parent(s): 8ac82f4

Upload 2 files

Browse files
Files changed (2) hide show
  1. app.py +165 -0
  2. requirements.txt +0 -0
app.py ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import cv2
3
+ import clip
4
+ import time
5
+ import torch
6
+ import warnings
7
+ import numpy as np
8
+ import streamlit as st
9
+ from PIL import Image, ImageDraw
10
+ from segment_anything import build_sam, SamAutomaticMaskGenerator
11
+
12
+ warnings.filterwarnings("ignore")
13
+
14
+ st.set_page_config(
15
+ page_title="CLIP+SAM - WebApp",
16
+ page_icon="✨",
17
+ layout="wide",
18
+ initial_sidebar_state="auto",
19
+ )
20
+
21
+ MODEL_CHECKPOINT = "model/sam_vit_h_4b8939.pth"
22
+ upload_path = "uploads/"
23
+
24
+
25
+ @st.cache_resource()
26
+ def mask_generate(MODEL_CHECKPOINT):
27
+ # Download the model weights to load them here
28
+ model_start_time = time.time()
29
+ mask_generator = SamAutomaticMaskGenerator(build_sam(checkpoint=MODEL_CHECKPOINT))
30
+ model_end_time = time.time()
31
+ print("-" * 50)
32
+ print(
33
+ f"Model downloaded successfully in {model_end_time - model_start_time} seconds."
34
+ )
35
+ return mask_generator
36
+
37
+
38
+ def generate_image_masks(image_path, mask_generator):
39
+ img_mask_start_time = time.time()
40
+ image = cv2.imread(image_path)
41
+ image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
42
+ masks = mask_generator.generate(image)
43
+ img_mask_end_time = time.time()
44
+ print("-" * 50)
45
+ print(
46
+ f"Image mask generated successfully in {img_mask_end_time - img_mask_start_time} seconds."
47
+ )
48
+ return masks
49
+
50
+
51
+ def convert_box_xywh_to_xyxy(box):
52
+ x1 = box[0]
53
+ y1 = box[1]
54
+ x2 = box[0] + box[2]
55
+ y2 = box[1] + box[3]
56
+ return [x1, y1, x2, y2]
57
+
58
+
59
+ def segment_image(image, segmentation_mask):
60
+ image_array = np.array(image)
61
+ segmented_image_array = np.zeros_like(image_array)
62
+ segmented_image_array[segmentation_mask] = image_array[segmentation_mask]
63
+ segmented_image = Image.fromarray(segmented_image_array)
64
+ black_image = Image.new("RGB", image.size, (0, 0, 0))
65
+ transparency_mask = np.zeros_like(segmentation_mask, dtype=np.uint8)
66
+ transparency_mask[segmentation_mask] = 255
67
+ transparency_mask_image = Image.fromarray(transparency_mask, mode="L")
68
+ black_image.paste(segmented_image, mask=transparency_mask_image)
69
+ return black_image
70
+
71
+
72
+ def load_CLIP():
73
+ # Load CLIP
74
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
75
+ print(f"Using {device.type} for inference...")
76
+ clip_start_time = time.time()
77
+ model, preprocess = clip.load("ViT-B/32", device=device)
78
+ clip_end_time = time.time()
79
+ print("-" * 50)
80
+ print(f"CLIP loaded successfully in {clip_end_time - clip_start_time} seconds.")
81
+ return model, preprocess, device
82
+
83
+
84
+ @torch.no_grad()
85
+ def retriev(elements: list[Image.Image], search_text: str) -> int:
86
+ model, preprocess, device = load_CLIP()
87
+ preprocessed_images = [preprocess(image).to(device) for image in elements]
88
+ tokenized_text = clip.tokenize([search_text]).to(device)
89
+ stacked_images = torch.stack(preprocessed_images)
90
+ image_features = model.encode_image(stacked_images)
91
+ text_features = model.encode_text(tokenized_text)
92
+ image_features /= image_features.norm(dim=-1, keepdim=True)
93
+ text_features /= text_features.norm(dim=-1, keepdim=True)
94
+ probs = 100.0 * image_features @ text_features.T
95
+ return probs[:, 0].softmax(dim=0)
96
+
97
+
98
+ def get_indices_of_values_above_threshold(values, threshold):
99
+ return [i for i, v in enumerate(values) if v > threshold]
100
+
101
+
102
+ mask_generator = mask_generate(MODEL_CHECKPOINT)
103
+
104
+ st.title("✨ CLIP + SAM 🏜")
105
+ st.info(" Let me help perform open vocabulary image segmentation. πŸ˜‰")
106
+ col_a, col_b = st.columns(2)
107
+
108
+ prompt = st.text_input("Enter your text", "pear")
109
+ image_path = st.file_uploader("Upload Image πŸš€", type=["png", "jpg", "bmp", "jpeg"])
110
+ if image_path is not None and (prompt is not None or len(prompt.strip()) != 0):
111
+ with open(os.path.join(upload_path, image_path.name), "wb") as f:
112
+ f.write((image_path).getbuffer())
113
+ uploaded_image_path = os.path.abspath(os.path.join(upload_path, image_path.name))
114
+ with st.spinner("Working... πŸ’«"):
115
+ # Cut out all masks
116
+ image = Image.open(uploaded_image_path)
117
+ cropped_boxes = []
118
+ masks = generate_image_masks(uploaded_image_path, mask_generator)
119
+
120
+ for mask in masks:
121
+ cropped_boxes.append(
122
+ segment_image(image, mask["segmentation"]).crop(
123
+ convert_box_xywh_to_xyxy(mask["bbox"])
124
+ )
125
+ )
126
+
127
+ scores = retriev(cropped_boxes, str(prompt))
128
+ indices = get_indices_of_values_above_threshold(scores, 0.05)
129
+
130
+ segmentation_masks = []
131
+
132
+ for seg_idx in indices:
133
+ segmentation_mask_image = Image.fromarray(
134
+ masks[seg_idx]["segmentation"].astype("uint8") * 255
135
+ )
136
+ segmentation_masks.append(segmentation_mask_image)
137
+
138
+ original_image = Image.open(uploaded_image_path)
139
+ overlay_image = Image.new("RGBA", image.size, (0, 0, 0, 0))
140
+ overlay_color = (255, 0, 0, 200)
141
+
142
+ draw = ImageDraw.Draw(overlay_image)
143
+ for segmentation_mask_image in segmentation_masks:
144
+ draw.bitmap((0, 0), segmentation_mask_image, fill=overlay_color)
145
+
146
+ result_image = Image.alpha_composite(
147
+ original_image.convert("RGBA"), overlay_image
148
+ )
149
+ np_image = np.array(result_image)
150
+ with st.container():
151
+ col1, col2 = st.columns(2)
152
+ with col1:
153
+ st.image(image, width=500)
154
+ st.success("Original Image")
155
+ with col2:
156
+ st.image(np_image, width=500)
157
+ st.success("Output based on CLIP+SAM")
158
+ else:
159
+ st.warning("⚠ Please upload your Image! 😯")
160
+
161
+
162
+ st.markdown(
163
+ "<br><hr><center>Made with ❀️ by <a href='mailto:ralhanprateek@gmail.com?subject=CLIP+SAM WebApp!&body=Please specify the issue you are facing with the app.'><strong>Prateek Ralhan</strong></a> with the help of [segment-anything](https://github.com/facebookresearch/segment-anything/tree/main) built by [Meta Research](https://github.com/facebookresearch) and [CLIP](https://github.com/openai/CLIP) built by [OpenAI](https://github.com/openai) ✨</center><hr>",
164
+ unsafe_allow_html=True,
165
+ )
requirements.txt ADDED
Binary file (12.4 kB). View file