Jacobmadwed commited on
Commit
f3abfa2
·
verified ·
1 Parent(s): e80eb3d

Update handler.py

Browse files
Files changed (1) hide show
  1. handler.py +325 -218
handler.py CHANGED
@@ -1,5 +1,6 @@
1
  import cv2
2
  import torch
 
3
  import numpy as np
4
  import PIL
5
  from PIL import Image
@@ -8,181 +9,250 @@ import diffusers
8
  from diffusers.utils import load_image
9
  from diffusers.models import ControlNetModel
10
  from diffusers.pipelines.controlnet.multicontrolnet import MultiControlNetModel
 
11
  from insightface.app import FaceAnalysis
12
  from style_template import styles
13
  from pipeline_stable_diffusion_xl_instantid_full import StableDiffusionXLInstantIDPipeline, draw_kps
14
  from controlnet_aux import OpenposeDetector
 
 
 
15
  import torch.nn.functional as F
16
  from torchvision.transforms import Compose
17
- import os
18
- from huggingface_hub import hf_hub_download
19
  import base64
20
  import io
21
- import json
22
 
23
  # global variable
 
24
  device = "cuda" if torch.cuda.is_available() else "cpu"
25
  dtype = torch.float16 if str(device).__contains__("cuda") else torch.float32
26
  STYLE_NAMES = list(styles.keys())
27
  DEFAULT_STYLE_NAME = "Spring Festival"
28
-
29
- class EndpointHandler:
30
- def __init__(self, model_dir):
31
- # Ensure the necessary files are downloaded
32
- controlnet_config = hf_hub_download(repo_id="InstantX/InstantID", filename="ControlNetModel/config.json", local_dir=os.path.join(model_dir, "checkpoints"))
33
- controlnet_model = hf_hub_download(repo_id="InstantX/InstantID", filename="ControlNetModel/diffusion_pytorch_model.safetensors", local_dir=os.path.join(model_dir, "checkpoints"))
34
- face_adapter = hf_hub_download(repo_id="InstantX/InstantID", filename="ip-adapter.bin", local_dir=os.path.join(model_dir, "checkpoints"))
35
-
36
- dir_path = os.path.join(model_dir, "models", "antelopev2")
37
- if not os.path.exists(dir_path):
38
- print(f"Model path {dir_path} does not exist. Attempting to download.")
39
- self.app = FaceAnalysis(name='antelopev2', root=model_dir, providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
40
- else:
41
- print(f"Model path {dir_path} exists. Skipping download.")
42
- self.app = FaceAnalysis(name='antelopev2', root=model_dir, providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
43
-
44
- self.app.prepare(ctx_id=0, det_size=(640, 640))
45
- openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
46
-
47
- # Path to InstantID models
48
- controlnet_path = os.path.join(model_dir, "checkpoints", "ControlNetModel")
49
-
50
- # Load pipeline face ControlNetModel
51
- self.controlnet_identitynet = ControlNetModel.from_pretrained(
52
- controlnet_path, torch_dtype=dtype
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
53
  )
54
-
55
- # controlnet-pose
56
- controlnet_pose_model = "thibaud/controlnet-openpose-sdxl-1.0"
57
- controlnet_canny_model = "diffusers/controlnet-canny-sdxl-1.0"
58
-
59
- controlnet_pose = ControlNetModel.from_pretrained(
60
- controlnet_pose_model, torch_dtype=dtype
61
- ).to(device)
62
- controlnet_canny = ControlNetModel.from_pretrained(
63
- controlnet_canny_model, torch_dtype=dtype
64
- ).to(device)
65
-
66
- def get_canny_image(image, t1=100, t2=200):
67
- image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
68
- edges = cv2.Canny(image, t1, t2)
69
- return Image.fromarray(edges, "L")
70
-
71
- self.controlnet_map = {
72
- "pose": controlnet_pose,
73
- "canny": controlnet_canny
74
- }
75
-
76
- self.controlnet_map_fn = {
77
- "pose": openpose,
78
- "canny": get_canny_image
79
- }
80
-
81
- pretrained_model_name_or_path = "wangqixun/YamerMIX_v8"
82
-
83
- self.pipe = StableDiffusionXLInstantIDPipeline.from_pretrained(
84
- pretrained_model_name_or_path,
85
- controlnet=[self.controlnet_identitynet],
86
- torch_dtype=dtype,
87
- safety_checker=None,
88
- feature_extractor=None,
89
- ).to(device)
90
-
91
- self.pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(
92
- self.pipe.scheduler.config
93
  )
94
 
95
- # load and disable LCM
96
- self.pipe.load_lora_weights("latent-consistency/lcm-lora-sdxl")
97
- self.pipe.disable_lora()
98
-
99
- self.pipe.cuda()
100
- self.pipe.load_ip_adapter_instantid(face_adapter)
101
- self.pipe.image_proj_model.to("cuda")
102
- self.pipe.unet.to("cuda")
103
-
104
- def __call__(self, data):
105
-
106
- def convert_from_cv2_to_image(img: np.ndarray) -> Image:
107
- return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
108
-
109
- def convert_from_image_to_cv2(img: Image) -> np.ndarray:
110
- return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
111
-
112
- def resize_img(
113
- input_image,
114
- max_side=1280,
115
- min_side=1024,
116
- size=None,
117
- pad_to_max_side=False,
118
- mode=PIL.Image.BILINEAR,
119
- base_pixel_number=64,
120
- ):
121
- w, h = input_image.size
122
- if size is not None:
123
- w_resize_new, h_resize_new = size
124
- else:
125
- ratio = min_side / min(h, w)
126
- w, h = round(ratio * w), round(ratio * h)
127
- ratio = max_side / max(h, w)
128
- input_image = input_image.resize([round(ratio * w), round(ratio * h)], mode)
129
- w_resize_new = (round(ratio * w) // base_pixel_number) * base_pixel_number
130
- h_resize_new = (round(ratio * h) // base_pixel_number) * base_pixel_number
131
- input_image = input_image.resize([w_resize_new, h_resize_new], mode)
132
-
133
- if pad_to_max_side:
134
- res = np.ones([max_side, max_side, 3], dtype=np.uint8) * 255
135
- offset_x = (max_side - w_resize_new) // 2
136
- offset_y = (max_side - h_resize_new) // 2
137
- res[
138
- offset_y : offset_y + h_resize_new, offset_x : offset_x + w_resize_new
139
- ] = np.array(input_image)
140
- input_image = Image.fromarray(res)
141
- return input_image
142
-
143
- def apply_style(
144
- style_name: str, positive: str, negative: str = ""
145
- ) -> Tuple[str, str]:
146
- p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME])
147
- return p.replace("{prompt}", positive), n + " " + negative
148
-
149
- def load_image_from_base64(base64_str):
150
- image_data = base64.b64decode(base64_str)
151
- image = Image.open(io.BytesIO(image_data))
152
- return image
153
-
154
- # Debug: Print incoming data
155
- print("Incoming data:", data)
156
-
157
- face_image_base64 = data.pop("face_image_base64", None)
158
- if face_image_base64 is not None:
159
- print("Using provided base64 image")
160
- face_image = load_image_from_base64(face_image_base64)
161
- else:
162
- print("No base64 image provided, using default path or URL")
163
- face_image_path = data.pop("face_image_path", "raw/input.jpg")
164
- face_image = load_image(face_image_path)
165
-
166
- # Debug: Print the used face image
167
- face_image.show(title="Debug Face Image")
168
-
169
- pose_image_path = data.pop("pose_image_path", None)
170
- style_name = data.pop("style_name", DEFAULT_STYLE_NAME)
171
- prompt = data.pop("inputs", "a man flying in the sky in Mars")
172
- negative_prompt = data.pop("negative_prompt", "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green")
173
- identitynet_strength_ratio = data.pop("identitynet_strength_ratio", 0.8)
174
- adapter_strength_ratio = data.pop("adapter_strength_ratio", 0.8)
175
- pose_strength = data.pop("pose_strength", 0.5)
176
- canny_strength = data.pop("canny_strength", 0.3)
177
- num_steps = data.pop("num_steps", 20)
178
- guidance_scale = data.pop("guidance_scale", 5.0)
179
- controlnet_selection = data.pop("controlnet_selection", ["pose", "canny"])
180
- scheduler = data.pop("scheduler", "EulerDiscreteScheduler")
181
- enable_fast_inference = data.pop("enable_fast_inference", False)
182
- enhance_non_face_region = data.pop("enhance_non_face_region", False)
183
- seed = data.pop("seed", 42)
184
-
185
- self.pipe.disable_lora()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
186
  scheduler_class_name = scheduler.split("-")[0]
187
 
188
  add_kwargs = {}
@@ -191,86 +261,123 @@ class EndpointHandler:
191
  if len(scheduler.split("-")) > 2:
192
  add_kwargs["algorithm_type"] = "sde-dpmsolver++"
193
  scheduler = getattr(diffusers, scheduler_class_name)
194
- self.pipe.scheduler = scheduler.from_config(self.pipe.scheduler.config, **add_kwargs)
 
 
 
 
 
 
195
 
196
- # apply the style template
197
- prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
198
 
199
- face_image = resize_img(face_image, max_side=1024)
200
- face_image_cv2 = convert_from_image_to_cv2(face_image)
201
- height, width, _ = face_image_cv2.shape
 
202
 
203
- # Extract face features
204
- face_info = self.app.get(face_image_cv2)
205
 
206
- face_info = sorted(
207
- face_info,
208
- key=lambda x: (x["bbox"][2] - x["bbox"][0]) * x["bbox"][3] - x["bbox"][1],
209
- )[
210
- -1
211
- ] # only use the maximum face
212
- face_emb = face_info["embedding"]
213
- face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info["kps"])
214
- img_controlnet = face_image
215
- if pose_image_path is not None:
216
- pose_image = load_image(pose_image_path)
217
- pose_image = resize_img(pose_image, max_side=1024)
218
- img_controlnet = pose_image
219
- pose_image_cv2 = convert_from_image_to_cv2(pose_image)
220
 
221
- face_info = self.app.get(pose_image_cv2)
 
 
 
 
 
 
 
 
222
 
223
- face_info = face_info[-1]
224
- face_kps = draw_kps(pose_image, face_info["kps"])
225
 
226
- width, height = face_kps.size
 
227
 
 
 
 
 
 
 
228
  control_mask = np.zeros([height, width, 3])
229
  x1, y1, x2, y2 = face_info["bbox"]
230
  x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
231
  control_mask[y1:y2, x1:x2] = 255
232
  control_mask = Image.fromarray(control_mask.astype(np.uint8))
 
 
233
 
 
234
  controlnet_scales = {
235
  "pose": pose_strength,
236
- "canny": canny_strength
 
237
  }
238
- self.pipe.controlnet = MultiControlNetModel(
239
- [self.controlnet_identitynet]
240
- + [self.controlnet_map[s] for s in controlnet_selection]
241
  )
242
  control_scales = [float(identitynet_strength_ratio)] + [
243
  controlnet_scales[s] for s in controlnet_selection
244
  ]
245
  control_images = [face_kps] + [
246
- self.controlnet_map_fn[s](img_controlnet).resize((width, height))
247
- for s in controlnet_selection
248
  ]
249
-
250
- generator = torch.Generator(device=device).manual_seed(seed)
251
-
252
- print("Start inference...")
253
- print(f"[Debug] Prompt: {prompt}, \n[Debug] Neg Prompt: {negative_prompt}")
254
-
255
- self.pipe.set_ip_adapter_scale(adapter_strength_ratio)
256
- images = self.pipe(
257
- prompt=prompt,
258
- negative_prompt=negative_prompt,
259
- image_embeds=face_emb,
260
- image=control_images,
261
- control_mask=control_mask,
262
- controlnet_conditioning_scale=control_scales,
263
- num_inference_steps=num_steps,
264
- guidance_scale=guidance_scale,
265
- height=height,
266
- width=width,
267
- generator=generator,
268
- ).images
269
-
270
- # Convert the output image to base64
271
- buffered = io.BytesIO()
272
- images[0].save(buffered, format="JPEG")
273
- img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
274
-
275
- return {"generated_image_base64": img_str}
276
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import cv2
2
  import torch
3
+ import random
4
  import numpy as np
5
  import PIL
6
  from PIL import Image
 
9
  from diffusers.utils import load_image
10
  from diffusers.models import ControlNetModel
11
  from diffusers.pipelines.controlnet.multicontrolnet import MultiControlNetModel
12
+ from huggingface_hub import hf_hub_download
13
  from insightface.app import FaceAnalysis
14
  from style_template import styles
15
  from pipeline_stable_diffusion_xl_instantid_full import StableDiffusionXLInstantIDPipeline, draw_kps
16
  from controlnet_aux import OpenposeDetector
17
+ import gradio as gr
18
+ from depth_anything.dpt import DepthAnything
19
+ from depth_anything.util.transform import Resize, NormalizeImage, PrepareForNet
20
  import torch.nn.functional as F
21
  from torchvision.transforms import Compose
 
 
22
  import base64
23
  import io
 
24
 
25
  # global variable
26
+ MAX_SEED = np.iinfo(np.int32).max
27
  device = "cuda" if torch.cuda.is_available() else "cpu"
28
  dtype = torch.float16 if str(device).__contains__("cuda") else torch.float32
29
  STYLE_NAMES = list(styles.keys())
30
  DEFAULT_STYLE_NAME = "Spring Festival"
31
+ enable_lcm_arg = False
32
+
33
+ # download checkpoints
34
+ hf_hub_download(repo_id="InstantX/InstantID", filename="ControlNetModel/config.json", local_dir="./checkpoints")
35
+ hf_hub_download(repo_id="InstantX/InstantID", filename="ControlNetModel/diffusion_pytorch_model.safetensors", local_dir="./checkpoints")
36
+ hf_hub_download(repo_id="InstantX/InstantID", filename="ip-adapter.bin", local_dir="./checkpoints")
37
+
38
+ # Load face encoder
39
+ app = FaceAnalysis(name="antelopev2", root="./", providers=["CPUExecutionProvider"])
40
+ app.prepare(ctx_id=0, det_size=(640, 640))
41
+
42
+ openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
43
+ depth_anything = DepthAnything.from_pretrained('LiheYoung/depth_anything_vitl14').to(device).eval()
44
+
45
+ transform = Compose([
46
+ Resize(width=518, height=518, resize_target=False, keep_aspect_ratio=True, ensure_multiple_of=14, resize_method='lower_bound', image_interpolation_method=cv2.INTER_CUBIC),
47
+ NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
48
+ PrepareForNet(),
49
+ ])
50
+
51
+ # Path to InstantID models
52
+ face_adapter = "./checkpoints/ip-adapter.bin"
53
+ controlnet_path = "./checkpoints/ControlNetModel"
54
+
55
+ # Load pipeline face ControlNetModel
56
+ controlnet_identitynet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=dtype)
57
+
58
+ # controlnet-pose/canny/depth
59
+ controlnet_pose_model = "thibaud/controlnet-openpose-sdxl-1.0"
60
+ controlnet_canny_model = "diffusers/controlnet-canny-sdxl-1.0"
61
+ controlnet_depth_model = "diffusers/controlnet-depth-sdxl-1.0-small"
62
+
63
+ controlnet_pose = ControlNetModel.from_pretrained(controlnet_pose_model, torch_dtype=dtype).to(device)
64
+ controlnet_canny = ControlNetModel.from_pretrained(controlnet_canny_model, torch_dtype=dtype).to(device)
65
+ controlnet_depth = ControlNetModel.from_pretrained(controlnet_depth_model, torch_dtype=dtype).to(device)
66
+
67
+ def get_depth_map(image):
68
+ image = np.array(image) / 255.0
69
+ h, w = image.shape[:2]
70
+ image = transform({'image': image})['image']
71
+ image = torch.from_numpy(image).unsqueeze(0).to("cuda")
72
+ with torch.no_grad():
73
+ depth = depth_anything(image)
74
+ depth = F.interpolate(depth[None], (h, w), mode='bilinear', align_corners=False)[0, 0]
75
+ depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0
76
+ depth = depth.cpu().numpy().astype(np.uint8)
77
+ depth_image = Image.fromarray(depth)
78
+ return depth_image
79
+
80
+ def get_canny_image(image, t1=100, t2=200):
81
+ image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
82
+ edges = cv2.Canny(image, t1, t2)
83
+ return Image.fromarray(edges, "L")
84
+
85
+ controlnet_map = {
86
+ "pose": controlnet_pose,
87
+ "canny": controlnet_canny,
88
+ "depth": controlnet_depth,
89
+ }
90
+ controlnet_map_fn = {
91
+ "pose": openpose,
92
+ "canny": get_canny_image,
93
+ "depth": get_depth_map,
94
+ }
95
+
96
+ pretrained_model_name_or_path = "wangqixun/YamerMIX_v8"
97
+
98
+ pipe = StableDiffusionXLInstantIDPipeline.from_pretrained(
99
+ pretrained_model_name_or_path,
100
+ controlnet=[controlnet_identitynet],
101
+ torch_dtype=dtype,
102
+ safety_checker=None,
103
+ feature_extractor=None,
104
+ ).to(device)
105
+
106
+ pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(pipe.scheduler.config)
107
+
108
+ # load and disable LCM
109
+ pipe.load_lora_weights("latent-consistency/lcm-lora-sdxl")
110
+ pipe.disable_lora()
111
+
112
+ pipe.cuda()
113
+ pipe.load_ip_adapter_instantid(face_adapter)
114
+ pipe.image_proj_model.to("cuda")
115
+ pipe.unet.to("cuda")
116
+
117
+ def toggle_lcm_ui(value):
118
+ if value:
119
+ return (
120
+ gr.update(minimum=0, maximum=100, step=1, value=5),
121
+ gr.update(minimum=0.1, maximum=20.0, step=0.1, value=1.5),
122
  )
123
+ else:
124
+ return (
125
+ gr.update(minimum=5, maximum=100, step=1, value=30),
126
+ gr.update(minimum=0.1, maximum=20.0, step=0.1, value=5),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
127
  )
128
 
129
+ def randomize_seed_fn(seed: int, randomize_seed: bool) -> int:
130
+ if randomize_seed:
131
+ seed = random.randint(0, MAX_SEED)
132
+ return seed
133
+
134
+ def remove_tips():
135
+ return gr.update(visible=False)
136
+
137
+ def get_example():
138
+ case = [
139
+ [
140
+ "./examples/yann-lecun_resize.jpg",
141
+ None,
142
+ "a man",
143
+ "Spring Festival",
144
+ "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
145
+ ],
146
+ [
147
+ "./examples/musk_resize.jpeg",
148
+ "./examples/poses/pose2.jpg",
149
+ "a man flying in the sky in Mars",
150
+ "Mars",
151
+ "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
152
+ ],
153
+ [
154
+ "./examples/sam_resize.png",
155
+ "./examples/poses/pose4.jpg",
156
+ "a man doing a silly pose wearing a suite",
157
+ "Jungle",
158
+ "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, gree",
159
+ ],
160
+ [
161
+ "./examples/schmidhuber_resize.png",
162
+ "./examples/poses/pose3.jpg",
163
+ "a man sit on a chair",
164
+ "Neon",
165
+ "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
166
+ ],
167
+ [
168
+ "./examples/kaifu_resize.png",
169
+ "./examples/poses/pose.jpg",
170
+ "a man",
171
+ "Vibrant Color",
172
+ "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, photo, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green",
173
+ ],
174
+ ]
175
+ return case
176
+
177
+ def run_for_examples(face_file, pose_file, prompt, style, negative_prompt):
178
+ return generate_image(
179
+ face_file,
180
+ pose_file,
181
+ prompt,
182
+ negative_prompt,
183
+ style,
184
+ 20, # num_steps
185
+ 0.8, # identitynet_strength_ratio
186
+ 0.8, # adapter_strength_ratio
187
+ 0.4, # pose_strength
188
+ 0.3, # canny_strength
189
+ 0.5, # depth_strength
190
+ ["pose", "canny"], # controlnet_selection
191
+ 5.0, # guidance_scale
192
+ 42, # seed
193
+ "EulerDiscreteScheduler", # scheduler
194
+ False, # enable_LCM
195
+ True, # enable_Face_Region
196
+ )
197
+
198
+ def convert_from_cv2_to_image(img: np.ndarray) -> Image:
199
+ return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
200
+
201
+ def convert_from_image_to_cv2(img: Image) -> np.ndarray:
202
+ return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
203
+
204
+ def resize_img(input_image, max_side=1280, min_side=1024, size=None, pad_to_max_side=False, mode=PIL.Image.BILINEAR, base_pixel_number=64):
205
+ w, h = input_image.size
206
+ if size is not None:
207
+ w_resize_new, h_resize_new = size
208
+ else:
209
+ ratio = min_side / min(h, w)
210
+ w, h = round(ratio * w), round(ratio * h)
211
+ ratio = max_side / max(h, w)
212
+ input_image = input_image.resize([round(ratio * w), round(ratio * h)], mode)
213
+ w_resize_new = (round(ratio * w) // base_pixel_number) * base_pixel_number
214
+ h_resize_new = (round(ratio * h) // base_pixel_number) * base_pixel_number
215
+ input_image = input_image.resize([w_resize_new, h_resize_new], mode)
216
+
217
+ if pad_to_max_side:
218
+ res = np.ones([max_side, max_side, 3], dtype=np.uint8) * 255
219
+ offset_x = (max_side - w_resize_new) // 2
220
+ offset_y = (max_side - h_resize_new) // 2
221
+ res[offset_y : offset_y + h_resize_new, offset_x : offset_x + w_resize_new] = np.array(input_image)
222
+ input_image = Image.fromarray(res)
223
+ return input_image
224
+
225
+ def apply_style(style_name: str, positive: str, negative: str = "") -> Tuple[str, str]:
226
+ p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME])
227
+ return p.replace("{prompt}", positive), n + " " + negative
228
+
229
+ @spaces.GPU
230
+ def generate_image(
231
+ face_image_path,
232
+ pose_image_path,
233
+ prompt,
234
+ negative_prompt,
235
+ style_name,
236
+ num_steps,
237
+ identitynet_strength_ratio,
238
+ adapter_strength_ratio,
239
+ pose_strength,
240
+ canny_strength,
241
+ depth_strength,
242
+ controlnet_selection,
243
+ guidance_scale,
244
+ seed,
245
+ scheduler,
246
+ enable_LCM,
247
+ enhance_face_region,
248
+ progress=gr.Progress(track_tqdm=True),
249
+ ):
250
+
251
+ if enable_LCM:
252
+ pipe.scheduler = diffusers.LCMScheduler.from_config(pipe.scheduler.config)
253
+ pipe.enable_lora()
254
+ else:
255
+ pipe.disable_lora()
256
  scheduler_class_name = scheduler.split("-")[0]
257
 
258
  add_kwargs = {}
 
261
  if len(scheduler.split("-")) > 2:
262
  add_kwargs["algorithm_type"] = "sde-dpmsolver++"
263
  scheduler = getattr(diffusers, scheduler_class_name)
264
+ pipe.scheduler = scheduler.from_config(pipe.scheduler.config, **add_kwargs)
265
+
266
+ if face_image_path is None:
267
+ raise gr.Error(f"Cannot find any input face image! Please upload the face image")
268
+
269
+ if prompt is None:
270
+ prompt = "a person"
271
 
272
+ # apply the style template
273
+ prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
274
 
275
+ face_image = load_image(face_image_path)
276
+ face_image = resize_img(face_image, max_side=1024)
277
+ face_image_cv2 = convert_from_image_to_cv2(face_image)
278
+ height, width, _ = face_image_cv2.shape
279
 
280
+ # Extract face features
281
+ face_info = app.get(face_image_cv2)
282
 
283
+ if len(face_info) == 0:
284
+ raise gr.Error(f"Unable to detect a face in the image. Please upload a different photo with a clear face.")
 
 
 
 
 
 
 
 
 
 
 
 
285
 
286
+ face_info = sorted(face_info, key=lambda x: (x["bbox"][2] - x["bbox"][0]) * x["bbox"][3] - x["bbox"][1])[-1] # only use the maximum face
287
+ face_emb = face_info["embedding"]
288
+ face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info["kps"])
289
+ img_controlnet = face_image
290
+ if pose_image_path is not None:
291
+ pose_image = load_image(pose_image_path)
292
+ pose_image = resize_img(pose_image, max_side=1024)
293
+ img_controlnet = pose_image
294
+ pose_image_cv2 = convert_from_image_to_cv2(pose_image)
295
 
296
+ face_info = app.get(pose_image_cv2)
 
297
 
298
+ if len(face_info) == 0:
299
+ raise gr.Error(f"Cannot find any face in the reference image! Please upload another person image")
300
 
301
+ face_info = face_info[-1]
302
+ face_kps = draw_kps(pose_image, face_info["kps"])
303
+
304
+ width, height = face_kps.size
305
+
306
+ if enhance_face_region:
307
  control_mask = np.zeros([height, width, 3])
308
  x1, y1, x2, y2 = face_info["bbox"]
309
  x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
310
  control_mask[y1:y2, x1:x2] = 255
311
  control_mask = Image.fromarray(control_mask.astype(np.uint8))
312
+ else:
313
+ control_mask = None
314
 
315
+ if len(controlnet_selection) > 0:
316
  controlnet_scales = {
317
  "pose": pose_strength,
318
+ "canny": canny_strength,
319
+ "depth": depth_strength,
320
  }
321
+ pipe.controlnet = MultiControlNetModel(
322
+ [controlnet_identitynet] + [controlnet_map[s] for s in controlnet_selection]
 
323
  )
324
  control_scales = [float(identitynet_strength_ratio)] + [
325
  controlnet_scales[s] for s in controlnet_selection
326
  ]
327
  control_images = [face_kps] + [
328
+ controlnet_map_fn[s](img_controlnet).resize((width, height)) for s in controlnet_selection
 
329
  ]
330
+ else:
331
+ pipe.controlnet = controlnet_identitynet
332
+ control_scales = float(identitynet_strength_ratio)
333
+ control_images = face_kps
334
+
335
+ generator = torch.Generator(device=device).manual_seed(seed)
336
+
337
+ print("Start inference...")
338
+ print(f"[Debug] Prompt: {prompt}, \n[Debug] Neg Prompt: {negative_prompt}")
339
+
340
+ pipe.set_ip_adapter_scale(adapter_strength_ratio)
341
+ images = pipe(
342
+ prompt=prompt,
343
+ negative_prompt=negative_prompt,
344
+ image_embeds=face_emb,
345
+ image=control_images,
346
+ control_mask=control_mask,
347
+ controlnet_conditioning_scale=control_scales,
348
+ num_inference_steps=num_steps,
349
+ guidance_scale=guidance_scale,
350
+ height=height,
351
+ width=width,
352
+ generator=generator,
353
+ ).images
354
+
355
+ # Convert the output image to base64
356
+ buffered = io.BytesIO()
357
+ images[0].save(buffered, format="JPEG")
358
+ img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
359
+
360
+ return {"generated_image_base64": img_str}
361
+
362
+ # Example usage
363
+ if __name__ == "__main__":
364
+ face_image_path = "path_to_face_image.jpg" # Replace with actual path or URL
365
+ pose_image_path = None # Replace with actual path or URL if available
366
+ prompt = "Create an extremely sharp, vibrant, and dynamic abstract photo of a person. The subject should be standing straight, wearing a modern outfit composed of multicolored, geometric patterns with no logos. Use bold and contrasting colors like red, blue, yellow, green, and orange. The background should feature an array of abstract geometric shapes and lines, creating a sense of depth and movement. Draw inspiration from abstract art and pop art, incorporating elements of cubism. Use natural lighting to highlight the subject’s features and the texture of the clothing, ensuring a polished and professional look. Emphasize crisp details, high resolution, and clarity to achieve an extremely sharp image."
367
+ negative_prompt = "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, glitch, deformed, mutated, cross-eyed, ugly, no log, disfigured, blurry, grainy"
368
+ style_name = "(No style)"
369
+ num_steps = 50
370
+ identitynet_strength_ratio = 0.8
371
+ adapter_strength_ratio = 0.8
372
+ pose_strength = 0
373
+ canny_strength = 0.5
374
+ depth_strength = 0
375
+ controlnet_selection = ["canny"]
376
+ guidance_scale = 2.0
377
+ seed = 96
378
+ scheduler = "DPMSolverMultistepScheduler"
379
+ enable_LCM = True
380
+ enhance_face_region = True
381
+
382
+ result = generate_image(face_image_path, pose_image_path, prompt, negative_prompt, style_name, num_steps, identitynet_strength_ratio, adapter_strength_ratio, pose_strength, canny_strength, depth_strength, controlnet_selection, guidance_scale, seed, scheduler, enable_LCM, enhance_face_region)
383
+ print(result)