Spaces:
Running on Zero
Running on Zero
Download temp_DRCT_reference/data/diffusion_rec.py from Thiru1122/Multimodel_setup: direct link, hf CLI and curl.
- Browser
- Download file 9.81 kB
-
https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/temp_DRCT_reference/data/diffusion_rec.py
- Command line
-
hf download hf://spaces/Thiru1122/Multimodel_setup/temp_DRCT_reference/data/diffusion_rec.py
-
curl -L -o diffusion_rec.py https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/temp_DRCT_reference/data/diffusion_rec.py
9.81 kB
| from diffusers import StableDiffusionInpaintPipeline, AutoPipelineForInpainting | |
| import torch | |
| from PIL import Image | |
| import numpy as np | |
| import cv2 | |
| import random | |
| import os | |
| import glob | |
| from tqdm import tqdm | |
| import albumentations as A | |
| import gc | |
| GenImage_LIST = [ | |
| 'stable_diffusion_v_1_4/imagenet_ai_0419_sdv4', 'stable_diffusion_v_1_5/imagenet_ai_0424_sdv5', | |
| 'Midjourney/imagenet_midjourney', 'ADM/imagenet_ai_0508_adm', 'wukong/imagenet_ai_0424_wukong', | |
| 'glide/imagenet_glide', 'VQDM/imagenet_ai_0419_vqdm', 'BigGAN/imagenet_ai_0419_biggan' | |
| ] | |
| DRCT_2M_LIST = [ | |
| 'ldm-text2im-large-256', 'stable-diffusion-v1-4', 'stable-diffusion-v1-5', 'stable-diffusion-2-1', | |
| 'stable-diffusion-xl-base-1.0', 'stable-diffusion-xl-refiner-1.0', 'sd-turbo', 'sdxl-turbo', | |
| 'lcm-lora-sdv1-5', 'lcm-lora-sdxl', 'sd-controlnet-canny', | |
| 'sd21-controlnet-canny', 'controlnet-canny-sdxl-1.0', 'stable-diffusion-inpainting', | |
| 'stable-diffusion-2-inpainting', 'stable-diffusion-xl-1.0-inpainting-0.1', | |
| ] | |
| def create_crop_transforms(height=224, width=224): | |
| aug_list = [ | |
| A.PadIfNeeded(min_height=height, min_width=width, border_mode=cv2.BORDER_CONSTANT, value=0), | |
| A.CenterCrop(height=height, width=width) | |
| ] | |
| return A.Compose(aug_list) | |
| def set_seed(seed: int): | |
| torch.manual_seed(seed) | |
| torch.cuda.manual_seed_all(seed) | |
| torch.backends.cudnn.deterministic = True | |
| torch.backends.cudnn.benchmark = False | |
| np.random.seed(seed) | |
| random.seed(seed) | |
| def find_nearest_multiple(a, multiple=8): | |
| """ | |
| 找到最接近a的multiple倍数,且该倍数大于a | |
| """ | |
| n = a // multiple | |
| remainder = a % multiple | |
| if remainder == 0: | |
| # 如果a整除multiple,,那么n就是multiple倍数 | |
| return a | |
| else: | |
| # 否则,我们需要找到更大的一个multiple的倍数 | |
| return (n + 1) * multiple | |
| def pad_image_to_size(image, target_width=224, target_height=224, fill_value=255): | |
| """将图像填充为目标宽度和高度,使用指定的填充值(默认为255)""" | |
| height, width = image.shape[:2] | |
| if height < target_height: | |
| pad_height = target_height - height | |
| pad_top = pad_height // 2 | |
| pad_bottom = pad_height - pad_top | |
| else: | |
| pad_top = pad_bottom = 0 | |
| if width < target_width: | |
| pad_width = target_width - width | |
| pad_left = pad_width // 2 | |
| pad_right = pad_width - pad_left | |
| else: | |
| pad_left = pad_right = 0 | |
| padded_image = np.pad( | |
| image, | |
| ((pad_top, pad_bottom), (pad_left, pad_right), (0, 0)), | |
| mode="constant", | |
| constant_values=fill_value | |
| ) | |
| return padded_image | |
| def center_crop(image, crop_width, crop_height): | |
| height, width = image.shape[:2] | |
| # 计算裁剪区域的起始点和终点 | |
| if width > crop_width: | |
| start_x = (width - crop_width) // 2 | |
| end_x = start_x + crop_width | |
| else: | |
| start_x, end_x = 0, width | |
| if height > crop_height: | |
| start_y = (height - crop_height) // 2 | |
| end_y = start_y + crop_height | |
| else: | |
| start_y, end_y = 0, height | |
| # 使用数组切片实现 center crop | |
| cropped_image = image[start_y:end_y, start_x:end_x] | |
| if cropped_image.shape[0] < crop_height or cropped_image.shape[1] < crop_width: | |
| cropped_image = pad_image_to_size(cropped_image, target_width=crop_width, target_height=crop_width, | |
| fill_value=255) | |
| return cropped_image | |
| def stable_diffusion_inpainting(pipe, image, mask_image, prompt, steps=50, height=512, width=512, | |
| seed=2023, guidance_scale=7.5): | |
| set_seed(int(seed)) | |
| image_pil = Image.fromarray(image) | |
| mask_image_pil = Image.fromarray(mask_image).convert("L") | |
| # image and mask_image should be PIL images. | |
| # The mask structure is white for inpainting and black for keeping as is | |
| new_image = pipe(prompt=prompt, image=image_pil, mask_image=mask_image_pil, | |
| height=height, width=width, num_inference_steps=steps, | |
| guidance_scale=guidance_scale).images[0] | |
| return new_image | |
| def read_image(image_path, max_size=512): | |
| create_crop_transforms(height=224, width=224) | |
| image = cv2.imread(image_path) | |
| image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) | |
| # crop image | |
| height, width = image.shape[:2] | |
| height = height if height < max_size else max_size | |
| width = width if width < max_size else max_size | |
| transform = create_crop_transforms(height=height, width=width) | |
| image = transform(image=image)["image"] | |
| # 处理8的倍数 | |
| original_shape = image.shape | |
| new_height = find_nearest_multiple(original_shape[0], multiple=8) | |
| new_width = find_nearest_multiple(original_shape[1], multiple=8) | |
| new_image = np.zeros(shape=(new_height, new_width, 3), dtype=image.dtype) | |
| new_image[:original_shape[0], :original_shape[1]] = image | |
| mask_image = np.zeros_like(image) | |
| del transform | |
| del image | |
| gc.collect() | |
| return new_image, mask_image, original_shape | |
| def func(image_path, save_path, crop_save_path, step=50, max_size=1024): | |
| image, mask_image, original_shape = read_image(image_path, max_size) | |
| # print(image.shape, mask_image.shape, np.unique(mask_image)) | |
| new_image = stable_diffusion_inpainting(pipe, image, mask_image, prompt='', steps=step, | |
| height=image.shape[0], | |
| width=image.shape[1], | |
| seed=2023, guidance_scale=7.5) | |
| # 恢复原来的尺寸 | |
| new_image = new_image.crop(box=(0, 0, original_shape[1], original_shape[0])) | |
| # 保存结果 | |
| new_image.save(save_path) | |
| if not os.path.exists(crop_save_path): | |
| image = Image.fromarray(image).crop(box=(0, 0, original_shape[1], original_shape[0])) | |
| image.save(crop_save_path) | |
| if __name__ == '__main__': | |
| # load stable diffusion models | |
| device = 'cuda' if torch.cuda.is_available() else 'cpu' | |
| root = '/disk2/chenby/nas_dsw/dataset/AIGC_data' | |
| sd_model_names = ["runwayml/stable-diffusion-inpainting", | |
| "stabilityai/stable-diffusion-2-inpainting", | |
| "diffusers/stable-diffusion-xl-1.0-inpainting-0.1" | |
| ] | |
| index = 0 | |
| sd_model_name = sd_model_names[index] | |
| if 'xl' in sd_model_name: | |
| pipe = AutoPipelineForInpainting.from_pretrained( | |
| sd_model_name, | |
| torch_dtype=torch.float16, | |
| variant="fp16", | |
| safety_checker=None, | |
| requires_safety_checker=False, # 关闭安全审查机制 | |
| ) | |
| pipe.enable_xformers_memory_efficient_attention() | |
| # pipe.enable_model_cpu_offload() | |
| pipe = pipe.to(device) | |
| else: | |
| pipe = StableDiffusionInpaintPipeline.from_pretrained( | |
| sd_model_name, | |
| # revision="fp16", | |
| torch_dtype=torch.float16, | |
| safety_checker=None, | |
| requires_safety_checker=False, # 关闭安全审查机制 | |
| ) | |
| pipe.enable_xformers_memory_efficient_attention() | |
| # pipe.enable_model_cpu_offload() | |
| pipe = pipe.to(device) | |
| print(f"Load model successful:{sd_model_name}") | |
| # Create "SDv1-DR", "SDv2-DR" and "SDXL-DR" from MSCOCO dataset | |
| step = 50 | |
| phase = 'train' | |
| model_name = 'real' | |
| inpainting_dir = {0: 'full_inpainting', 1: 'full_inpainting2', 2: 'full_inpainting_xl'}[index] | |
| if step != 50: | |
| inpainting_dir = f'step{step}_{inpainting_dir}' | |
| image_root = f'{root}/MSCOCO/{phase}2017' | |
| save_root = f'{root}/DR/MSCOCO/{inpainting_dir}/{phase}2017' | |
| crop_root = None | |
| # Create reconstructed images for the DRCT-2M dataset. | |
| # step = 50 | |
| # phase = 'val' | |
| # model_index = 0 | |
| # model_name = DRCT_2M_LIST[model_index] | |
| # inpainting_dir = {0: 'full_inpainting', 1: 'full_inpainting2', 2: 'full_inpainting_xl'}[index] | |
| # if step != 50: | |
| # inpainting_dir = f'step{step}_{inpainting_dir}' | |
| # image_root = f'{root}/DRCT-2M/{model_name}/{phase}2017' | |
| # save_root = f'{root}/DR/DRCT-2M/{model_name}/{inpainting_dir}/{phase}2017' | |
| # crop_root = None | |
| # Create reconstructed images for the GenImage dataset. | |
| # step = 50 | |
| # phase = 'train' | |
| # label = 'ai' | |
| # inpainting_dir = {0: 'inpainting', 1: 'inpainting2', 2: 'inpainting_xl'}[index] | |
| # model_index = 0 | |
| # model_name = GenImage_LIST[model_index] | |
| # image_root = f'{root}/GenImage/{model_name}/{phase}/{label}' | |
| # save_root = f'{root}/DR/GenImage/{model_name}/{phase}/{label}/{inpainting_dir}' | |
| # crop_root = f'{root}/DR/GenImage/{model_name}/{phase}/{label}/crop' | |
| os.makedirs(save_root, exist_ok=True) | |
| if crop_root is not None: | |
| os.makedirs(crop_root, exist_ok=True) | |
| start_index, end_index = 0, 200000 | |
| image_paths = sorted(glob.glob(f"{image_root}/*.*"))[start_index:end_index] | |
| print(f'start_index:{start_index}, end_index:{end_index}, {len(image_paths)}, image_root:{model_name}') | |
| failed_num = 0 | |
| for image_path in tqdm(image_paths): | |
| image_name = os.path.basename(image_path).split('.')[0] | |
| save_path = os.path.join(save_root, image_name + '.png') | |
| crop_save_path = os.path.join(crop_root, image_name + '.png') | |
| if os.path.exists(save_path): | |
| if (crop_root is not None and os.path.exists(crop_save_path)) or crop_root is None: | |
| continue | |
| try: | |
| func(image_path, save_path, crop_save_path, step=step, max_size=1024) | |
| except: | |
| failed_num += 1 | |
| print(f'Failed to generate image in {image_path}.') | |
| print(f'Inference finished! start_index:{start_index}, end_index:{end_index}, model_id:{model_name}, failed_num:{failed_num}') |