Akshat Santhana Gopalan commited on
Commit
d0dbf6e
·
1 Parent(s): 4bba4a1

Add missing inference modules

Browse files
Files changed (1) hide show
  1. seggpt_engine.py +191 -0
seggpt_engine.py ADDED
@@ -0,0 +1,191 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn.functional as F
3
+ import numpy as np
4
+ import cv2
5
+
6
+ from PIL import Image
7
+
8
+
9
+ imagenet_mean = np.array([0.485, 0.456, 0.406])
10
+ imagenet_std = np.array([0.229, 0.224, 0.225])
11
+
12
+
13
+ class Cache(list):
14
+ def __init__(self, max_size=0):
15
+ super().__init__()
16
+ self.max_size = max_size
17
+
18
+ def append(self, x):
19
+ if self.max_size <= 0:
20
+ return
21
+ super().append(x)
22
+ if len(self) > self.max_size:
23
+ self.pop(0)
24
+
25
+
26
+ @torch.no_grad()
27
+ def run_one_image(img, tgt, model, device):
28
+ x = torch.tensor(img)
29
+ # make it a batch-like
30
+ x = torch.einsum('nhwc->nchw', x)
31
+
32
+ tgt = torch.tensor(tgt)
33
+ # make it a batch-like
34
+ tgt = torch.einsum('nhwc->nchw', tgt)
35
+
36
+ bool_masked_pos = torch.zeros(model.patch_embed.num_patches)
37
+ bool_masked_pos[model.patch_embed.num_patches//2:] = 1
38
+ bool_masked_pos = bool_masked_pos.unsqueeze(dim=0)
39
+ valid = torch.ones_like(tgt)
40
+
41
+ if model.seg_type == 'instance':
42
+ seg_type = torch.ones([valid.shape[0], 1])
43
+ else:
44
+ seg_type = torch.zeros([valid.shape[0], 1])
45
+
46
+ feat_ensemble = 0 if len(x) > 1 else -1
47
+ _, y, mask = model(x.float().to(device), tgt.float().to(device), bool_masked_pos.to(device), valid.float().to(device), seg_type.to(device), feat_ensemble)
48
+ y = model.unpatchify(y)
49
+ y = torch.einsum('nchw->nhwc', y).detach().cpu()
50
+
51
+ output = y[0, y.shape[1]//2:, :, :]
52
+ output = torch.clip((output * imagenet_std + imagenet_mean) * 255, 0, 255)
53
+ return output
54
+
55
+
56
+ def inference_image(model, device, img_path, img2_paths, tgt2_paths, out_path):
57
+ res, hres = 448, 448
58
+
59
+ image = Image.open(img_path).convert("RGB")
60
+ input_image = np.array(image)
61
+ size = image.size
62
+ image = np.array(image.resize((res, hres))) / 255.
63
+
64
+ image_batch, target_batch = [], []
65
+ for img2_path, tgt2_path in zip(img2_paths, tgt2_paths):
66
+ img2 = Image.open(img2_path).convert("RGB")
67
+ img2 = img2.resize((res, hres))
68
+ img2 = np.array(img2) / 255.
69
+
70
+ tgt2 = Image.open(tgt2_path).convert("RGB")
71
+ tgt2 = tgt2.resize((res, hres), Image.NEAREST)
72
+ tgt2 = np.array(tgt2) / 255.
73
+
74
+ tgt = tgt2 # tgt is not available
75
+ tgt = np.concatenate((tgt2, tgt), axis=0)
76
+ img = np.concatenate((img2, image), axis=0)
77
+
78
+ assert img.shape == (2*res, res, 3), f'{img.shape}'
79
+ # normalize by ImageNet mean and std
80
+ img = img - imagenet_mean
81
+ img = img / imagenet_std
82
+
83
+ assert tgt.shape == (2*res, res, 3), f'{img.shape}'
84
+ # normalize by ImageNet mean and std
85
+ tgt = tgt - imagenet_mean
86
+ tgt = tgt / imagenet_std
87
+
88
+ image_batch.append(img)
89
+ target_batch.append(tgt)
90
+
91
+ img = np.stack(image_batch, axis=0)
92
+ tgt = np.stack(target_batch, axis=0)
93
+ """### Run SegGPT on the image"""
94
+ # make random mask reproducible (comment out to make it change)
95
+ torch.manual_seed(2)
96
+ output = run_one_image(img, tgt, model, device)
97
+ output = F.interpolate(
98
+ output[None, ...].permute(0, 3, 1, 2),
99
+ size=[size[1], size[0]],
100
+ mode='nearest',
101
+ ).permute(0, 2, 3, 1)[0].numpy()
102
+ # output = Image.fromarray((input_image * (0.6 * output / 255 + 0.4)).astype(np.uint8))
103
+ # output.save(out_path)
104
+
105
+ # Optional: Save blended image
106
+ blended = Image.fromarray((input_image * (0.6 * output / 255 + 0.4)).astype(np.uint8))
107
+ blended.save(out_path)
108
+
109
+ # Compute binary mask from output
110
+ mask = (output.mean(axis=-1) > 50).astype(np.uint8) # Threshold may be tuned
111
+
112
+ # Return the mask array
113
+ return mask
114
+
115
+
116
+ def inference_video(model, device, vid_path, num_frames, img2_paths, tgt2_paths, out_path):
117
+ res, hres = 448, 448
118
+
119
+ cap = cv2.VideoCapture(vid_path)
120
+ fps = cap.get(cv2.CAP_PROP_FPS)
121
+ width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
122
+ height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
123
+ fourcc = cv2.VideoWriter_fourcc(*'mp4v')
124
+ video_writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height), True)
125
+
126
+ if img2_paths is None:
127
+ _, frame = cap.read()
128
+ img2 = Image.fromarray(frame[:, :, ::-1]).convert('RGB')
129
+ else:
130
+ img2 = Image.open(img2_paths[0]).convert("RGB")
131
+ img2 = img2.resize((res, hres))
132
+ img2 = np.array(img2) / 255.
133
+
134
+ tgt2 = Image.open(tgt2_paths[0]).convert("RGB")
135
+ tgt2 = tgt2.resize((res, hres), Image.NEAREST)
136
+ tgt2 = np.array(tgt2) / 255.
137
+
138
+ frames_cache, target_cache = Cache(num_frames), Cache(num_frames)
139
+
140
+ while True:
141
+ ret, frame = cap.read()
142
+ if not ret:
143
+ break
144
+
145
+ image_batch, target_batch = [], []
146
+ image = Image.fromarray(frame[:, :, ::-1]).convert('RGB')
147
+ input_image = np.array(image)
148
+ size = image.size
149
+ image = np.array(image.resize((res, hres))) / 255.
150
+
151
+ for prompt, target in zip([img2] + frames_cache, [tgt2] + target_cache):
152
+ tgt = target # tgt is not available
153
+ tgt = np.concatenate((target, tgt), axis=0)
154
+ img = np.concatenate((prompt, image), axis=0)
155
+
156
+ assert img.shape == (2*res, res, 3), f'{img.shape}'
157
+ # normalize by ImageNet mean and std
158
+ img = img - imagenet_mean
159
+ img = img / imagenet_std
160
+
161
+ assert tgt.shape == (2*res, res, 3), f'{img.shape}'
162
+ # normalize by ImageNet mean and std
163
+ tgt = tgt - imagenet_mean
164
+ tgt = tgt / imagenet_std
165
+
166
+ image_batch.append(img)
167
+ target_batch.append(tgt)
168
+
169
+ img = np.stack(image_batch, axis=0)
170
+ tgt = np.stack(target_batch, axis=0)
171
+
172
+ torch.manual_seed(2)
173
+ output = run_one_image(img, tgt, model, device)
174
+
175
+ frames_cache.append(image)
176
+ target_cache.append(
177
+ output.mean(-1) \
178
+ .gt(128).float() \
179
+ .unsqueeze(-1).expand(-1, -1, 3) \
180
+ .numpy()
181
+ )
182
+
183
+ output = F.interpolate(
184
+ output[None, ...].permute(0, 3, 1, 2),
185
+ size=[size[1], size[0]],
186
+ mode='nearest',
187
+ ).permute(0, 2, 3, 1)[0].numpy()
188
+ output = input_image * (0.6 * output / 255 + 0.4)
189
+ video_writer.write(np.ascontiguousarray(output.astype(np.uint8)[:, :, ::-1]))
190
+
191
+ video_writer.release()