Spaces:
Build error
Build error
kelvinbksoh commited on
Commit ·
fbef195
1
Parent(s): c3c7569
added feature for batch image generation
Browse files
app.py
CHANGED
|
@@ -151,10 +151,11 @@ def copy_output_to_input(output_image):
|
|
| 151 |
return output_image
|
| 152 |
|
| 153 |
# @spaces.GPU(duration=240)
|
| 154 |
-
def generate_image(input_image, semantic_drawing,
|
| 155 |
"""
|
| 156 |
Generate image using the model with adjustable prob_mask parameter.
|
| 157 |
"""
|
|
|
|
| 158 |
tfs = transforms.Compose([
|
| 159 |
transforms.ToPILImage(),
|
| 160 |
transforms.Resize((256, 256)),
|
|
@@ -197,29 +198,108 @@ def generate_image(input_image, semantic_drawing, prob_mask, diffusion_steps):
|
|
| 197 |
sample_fn = (
|
| 198 |
diffusion.p_sample_loop if not args.use_ddim else diffusion.ddim_sample_loop
|
| 199 |
)
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 212 |
with gr.Blocks() as demo:
|
| 213 |
gr.Markdown("# Image-to-Image Generation with DDPM")
|
| 214 |
|
| 215 |
with gr.Row():
|
| 216 |
-
with gr.Column():
|
| 217 |
input_image = gr.Image(
|
| 218 |
label="Input Image",
|
| 219 |
type="numpy",
|
| 220 |
container=True,
|
|
|
|
| 221 |
)
|
| 222 |
-
with gr.Column():
|
| 223 |
semantic_drawing = gr.ImageEditor(
|
| 224 |
label="Semantic Drawing",
|
| 225 |
container=True,
|
|
@@ -229,22 +309,26 @@ with gr.Blocks() as demo:
|
|
| 229 |
color_mode='fixed',
|
| 230 |
default_size=20
|
| 231 |
),
|
| 232 |
-
interactive=True
|
|
|
|
| 233 |
)
|
| 234 |
|
| 235 |
|
| 236 |
-
with gr.Row():
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
|
|
|
|
|
|
|
|
|
| 248 |
|
| 249 |
# with gr.Row():
|
| 250 |
# prob_mask_slider = gr.Slider(
|
|
@@ -264,23 +348,33 @@ with gr.Blocks() as demo:
|
|
| 264 |
# label="Diffusion Steps",
|
| 265 |
# info="Number of diffusion steps to use"
|
| 266 |
# )
|
| 267 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 268 |
with gr.Row():
|
| 269 |
generate_btn = gr.Button("Generate Image", size="large")
|
| 270 |
|
| 271 |
|
| 272 |
with gr.Row():
|
| 273 |
-
output_image = gr.Image(
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
)
|
|
|
|
|
|
|
| 280 |
with gr.Row():
|
| 281 |
use_as_input_btn = gr.Button("Use Output as New Input")
|
| 282 |
|
| 283 |
-
|
| 284 |
# Example images
|
| 285 |
gr.Examples(
|
| 286 |
examples=[
|
|
@@ -294,15 +388,24 @@ with gr.Blocks() as demo:
|
|
| 294 |
# Connect the generate button
|
| 295 |
generate_btn.click(
|
| 296 |
fn=generate_image,
|
| 297 |
-
inputs=[input_image, semantic_drawing],
|
| 298 |
outputs=output_image
|
| 299 |
)
|
| 300 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 301 |
use_as_input_btn.click(
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
|
|
|
|
|
|
| 306 |
|
| 307 |
# Connect the dropdown to the update function
|
| 308 |
label_dropdown.change(
|
|
|
|
| 151 |
return output_image
|
| 152 |
|
| 153 |
# @spaces.GPU(duration=240)
|
| 154 |
+
def generate_image(input_image, semantic_drawing, num_imgs):
|
| 155 |
"""
|
| 156 |
Generate image using the model with adjustable prob_mask parameter.
|
| 157 |
"""
|
| 158 |
+
print("num_imgs", num_imgs)
|
| 159 |
tfs = transforms.Compose([
|
| 160 |
transforms.ToPILImage(),
|
| 161 |
transforms.Resize((256, 256)),
|
|
|
|
| 198 |
sample_fn = (
|
| 199 |
diffusion.p_sample_loop if not args.use_ddim else diffusion.ddim_sample_loop
|
| 200 |
)
|
| 201 |
+
|
| 202 |
+
output_imgs = []
|
| 203 |
+
for _ in range(num_imgs): ## Naive batch generation
|
| 204 |
+
sample = sample_fn(
|
| 205 |
+
model,
|
| 206 |
+
(1, 3, 256, 256),
|
| 207 |
+
clip_denoised=args.clip_denoised,
|
| 208 |
+
model_kwargs=model_kwargs,
|
| 209 |
+
progress=True,
|
| 210 |
+
)
|
| 211 |
+
sample = (sample + 1) / 2.0
|
| 212 |
+
sample = sample.cpu().permute(0, 2, 3, 1)
|
| 213 |
+
output_img = sample.cpu().numpy()[0]#.transpose(1, 2, 0)
|
| 214 |
+
output_imgs.append(output_img)
|
| 215 |
+
|
| 216 |
+
return output_imgs # return twice: for output and new input
|
| 217 |
+
|
| 218 |
+
## Batching approach
|
| 219 |
+
## 1. Able to generate the inpainted mask with batches
|
| 220 |
+
## 2. Need to overlay the original image with the inpainted mask (TODO)
|
| 221 |
+
# import torch
|
| 222 |
+
# def generate_image(input_image, semantic_drawing, num_imgs: int):
|
| 223 |
+
# """
|
| 224 |
+
# Generate `num_imgs` images with the diffusion model.
|
| 225 |
+
# """
|
| 226 |
+
# # ------------ preprocessing ------------------------------------------------
|
| 227 |
+
# tfm = transforms.Compose([
|
| 228 |
+
# transforms.ToPILImage(),
|
| 229 |
+
# transforms.Resize((256, 256)),
|
| 230 |
+
# transforms.ToTensor(),
|
| 231 |
+
# transforms.Normalize([0.5]*3, [0.5]*3),
|
| 232 |
+
# ])
|
| 233 |
+
# diffusion_steps = 30
|
| 234 |
+
|
| 235 |
+
# # (1) RGB image → tensor in [-1,1]
|
| 236 |
+
# img = tfm(input_image).unsqueeze(0) # (1,3,256,256) in [-1, 1]
|
| 237 |
+
|
| 238 |
+
# # (2) semantic mask → one-hot (1,19,256,256)
|
| 239 |
+
# raw = semantic_drawing["composite"][:, :, :3]
|
| 240 |
+
# semantic_idx = np.zeros(raw.shape[:2], np.uint8)
|
| 241 |
+
# for k, color in label_color_mapping.items():
|
| 242 |
+
# semantic_idx[(raw == color).all(-1)] = k
|
| 243 |
+
# semantic_idx = torch.from_numpy(semantic_idx).unsqueeze(0).unsqueeze(0) # (1,1,256,256)
|
| 244 |
+
|
| 245 |
+
# one_hot = torch.zeros(1, 19, 256, 256, dtype=torch.float32)
|
| 246 |
+
# one_hot.scatter_(1, semantic_idx.long(), 1.0)
|
| 247 |
+
|
| 248 |
+
# # (3) build model kwargs, **repeat along batch dimension**
|
| 249 |
+
# # so each of the `num_imgs` samples is conditioned the same way.
|
| 250 |
+
# input_semantics = one_hot.repeat(num_imgs, 1, 1, 1) # (N,19,256,256)
|
| 251 |
+
# model_kwargs = {"y": input_semantics, "s": args.s}
|
| 252 |
+
|
| 253 |
+
# # ------------ diffusion ----------------------------------------------------
|
| 254 |
+
# diffusion = recreate_diffusion_with_steps(args, diffusion_steps)
|
| 255 |
+
|
| 256 |
+
# sample_fn = diffusion.ddim_sample_loop if args.use_ddim else diffusion.p_sample_loop
|
| 257 |
+
# samples = sample_fn(
|
| 258 |
+
# model,
|
| 259 |
+
# (num_imgs, 3, 256, 256), # output batch shape
|
| 260 |
+
# clip_denoised=args.clip_denoised,
|
| 261 |
+
# model_kwargs=model_kwargs,
|
| 262 |
+
# progress=True,
|
| 263 |
+
# ) # (N,3,256,256)
|
| 264 |
+
|
| 265 |
+
# # ------------ post-processing ---------------------------------------------
|
| 266 |
+
# samples = (samples + 1) / 2 # back to [0,1]
|
| 267 |
+
# samples = samples.permute(0, 2, 3, 1).cpu().numpy() # (N,256,256,3)
|
| 268 |
+
|
| 269 |
+
# # return a Python list of individual images (optional but convenient)
|
| 270 |
+
# return [samples[i] for i in range(num_imgs)]
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def put_clicked_into_input(evt: gr.SelectData):
|
| 274 |
+
"""
|
| 275 |
+
evt.value -> value associated with the clicked item
|
| 276 |
+
(a NumPy array *or* a list, gradio versions differ)
|
| 277 |
+
evt.index -> zero-based index of the thumbnail that was clicked
|
| 278 |
+
"""
|
| 279 |
+
clicked = evt.value
|
| 280 |
+
image = clicked['image']['path']
|
| 281 |
+
if isinstance(clicked, list): # safety for older gradio versions
|
| 282 |
+
clicked = clicked[evt.index]
|
| 283 |
+
return image # single image (NumPy array)
|
| 284 |
|
| 285 |
+
def put_selected_into_input(chosen_img):
|
| 286 |
+
print('chosen_img', chosen_img)
|
| 287 |
+
if chosen_img is None:
|
| 288 |
+
raise gr.Error("Please click an image in the gallery first.")
|
| 289 |
+
return chosen_img
|
| 290 |
+
|
| 291 |
with gr.Blocks() as demo:
|
| 292 |
gr.Markdown("# Image-to-Image Generation with DDPM")
|
| 293 |
|
| 294 |
with gr.Row():
|
| 295 |
+
with gr.Column(scale=1):
|
| 296 |
input_image = gr.Image(
|
| 297 |
label="Input Image",
|
| 298 |
type="numpy",
|
| 299 |
container=True,
|
| 300 |
+
height=350
|
| 301 |
)
|
| 302 |
+
with gr.Column(scale=1):
|
| 303 |
semantic_drawing = gr.ImageEditor(
|
| 304 |
label="Semantic Drawing",
|
| 305 |
container=True,
|
|
|
|
| 309 |
color_mode='fixed',
|
| 310 |
default_size=20
|
| 311 |
),
|
| 312 |
+
interactive=True,
|
| 313 |
+
height=350
|
| 314 |
)
|
| 315 |
|
| 316 |
|
| 317 |
+
with gr.Row(equal_height=True):
|
| 318 |
+
with gr.Column(scale=1):
|
| 319 |
+
label_dropdown = gr.Dropdown(
|
| 320 |
+
choices=[f"{label_to_name[k]}: {label_color_mapping[k]}" for k in label_color_mapping],
|
| 321 |
+
label="Select Label",
|
| 322 |
+
value=f"No information: {label_to_name[0]}",
|
| 323 |
+
|
| 324 |
+
)
|
| 325 |
+
with gr.Column(scale=1):
|
| 326 |
+
color_display = gr.ColorPicker(
|
| 327 |
+
label="Selected Color",
|
| 328 |
+
value="#000000",
|
| 329 |
+
interactive=False,
|
| 330 |
+
container=True
|
| 331 |
+
)
|
| 332 |
|
| 333 |
# with gr.Row():
|
| 334 |
# prob_mask_slider = gr.Slider(
|
|
|
|
| 348 |
# label="Diffusion Steps",
|
| 349 |
# info="Number of diffusion steps to use"
|
| 350 |
# )
|
| 351 |
+
with gr.Row():
|
| 352 |
+
num_imgs_slider = gr.Slider(
|
| 353 |
+
minimum=1, maximum=4, value=1, step=1,
|
| 354 |
+
label="Images to generate"
|
| 355 |
+
)
|
| 356 |
with gr.Row():
|
| 357 |
generate_btn = gr.Button("Generate Image", size="large")
|
| 358 |
|
| 359 |
|
| 360 |
with gr.Row():
|
| 361 |
+
# output_image = gr.Image(
|
| 362 |
+
# label="Generated Image",
|
| 363 |
+
# height=256,
|
| 364 |
+
# width=256,
|
| 365 |
+
# container=True,
|
| 366 |
+
# min_width=400
|
| 367 |
+
# )
|
| 368 |
+
output_image = gr.Gallery(
|
| 369 |
+
label="Generated images",
|
| 370 |
+
height="auto", columns=[4], # 4 columns looks nice up to 8 images
|
| 371 |
+
preview=False,
|
| 372 |
)
|
| 373 |
+
|
| 374 |
+
selected_img = gr.State(value=None)
|
| 375 |
with gr.Row():
|
| 376 |
use_as_input_btn = gr.Button("Use Output as New Input")
|
| 377 |
|
|
|
|
| 378 |
# Example images
|
| 379 |
gr.Examples(
|
| 380 |
examples=[
|
|
|
|
| 388 |
# Connect the generate button
|
| 389 |
generate_btn.click(
|
| 390 |
fn=generate_image,
|
| 391 |
+
inputs=[input_image, semantic_drawing, num_imgs_slider],
|
| 392 |
outputs=output_image
|
| 393 |
)
|
| 394 |
|
| 395 |
+
output_image.select(
|
| 396 |
+
fn=put_clicked_into_input,
|
| 397 |
+
inputs=None, # the gallery value is the *selected image*, not the list
|
| 398 |
+
outputs=selected_img # gr.Image expects a single ndarray → OK
|
| 399 |
+
)
|
| 400 |
+
|
| 401 |
+
|
| 402 |
use_as_input_btn.click(
|
| 403 |
+
fn=put_selected_into_input,
|
| 404 |
+
inputs=selected_img, # comes from our hidden State
|
| 405 |
+
outputs=input_image
|
| 406 |
+
)
|
| 407 |
+
|
| 408 |
+
|
| 409 |
|
| 410 |
# Connect the dropdown to the update function
|
| 411 |
label_dropdown.change(
|