Instructions to use Charles-Elena/InstantID with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use Charles-Elena/InstantID with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("Charles-Elena/InstantID", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
Commit ·
c9fb62c
1
Parent(s): 4073f23
added depth
Browse files- handler.py +39 -39
handler.py
CHANGED
|
@@ -43,21 +43,21 @@ class EndpointHandler():
|
|
| 43 |
self.app.prepare(ctx_id=0, det_size=(640, 640))
|
| 44 |
|
| 45 |
openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
|
| 62 |
face_adapter = f"/repository/checkpoints/ip-adapter.bin"
|
| 63 |
controlnet_path = f"/repository/checkpoints/ControlNetModel"
|
|
@@ -68,7 +68,7 @@ class EndpointHandler():
|
|
| 68 |
|
| 69 |
controlnet_pose_model = "thibaud/controlnet-openpose-sdxl-1.0"
|
| 70 |
controlnet_canny_model = "diffusers/controlnet-canny-sdxl-1.0"
|
| 71 |
-
|
| 72 |
|
| 73 |
controlnet_pose = ControlNetModel.from_pretrained(
|
| 74 |
controlnet_pose_model, torch_dtype=dtype
|
|
@@ -76,30 +76,30 @@ class EndpointHandler():
|
|
| 76 |
controlnet_canny = ControlNetModel.from_pretrained(
|
| 77 |
controlnet_canny_model, torch_dtype=dtype
|
| 78 |
).to(device)
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
|
| 83 |
-
|
| 84 |
|
| 85 |
-
|
| 86 |
|
| 87 |
-
|
| 88 |
|
| 89 |
-
|
| 90 |
-
|
| 91 |
|
| 92 |
-
|
| 93 |
-
|
| 94 |
|
| 95 |
-
|
| 96 |
-
|
| 97 |
|
| 98 |
-
|
| 99 |
|
| 100 |
-
|
| 101 |
|
| 102 |
-
|
| 103 |
|
| 104 |
def get_canny_image(image, t1=100, t2=200):
|
| 105 |
image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
|
|
@@ -109,13 +109,13 @@ class EndpointHandler():
|
|
| 109 |
self.controlnet_map = {
|
| 110 |
"pose": controlnet_pose,
|
| 111 |
"canny": controlnet_canny,
|
| 112 |
-
|
| 113 |
}
|
| 114 |
|
| 115 |
self.controlnet_map_fn = {
|
| 116 |
"pose": openpose,
|
| 117 |
"canny": get_canny_image,
|
| 118 |
-
|
| 119 |
}
|
| 120 |
|
| 121 |
pretrained_model_name_or_path = "wangqixun/YamerMIX_v8"
|
|
@@ -142,10 +142,10 @@ class EndpointHandler():
|
|
| 142 |
self.pipe.unet.to("cuda")
|
| 143 |
|
| 144 |
# if we need more parameters
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
|
| 150 |
identitynet_strength_ratio = 0.8
|
| 151 |
|
|
@@ -153,8 +153,8 @@ class EndpointHandler():
|
|
| 153 |
canny_strength = 0.3
|
| 154 |
depth_strength = 0.5
|
| 155 |
|
| 156 |
-
self.my_controlnet_selection = ["pose", "canny"]
|
| 157 |
-
|
| 158 |
|
| 159 |
controlnet_scales = {
|
| 160 |
"pose": pose_strength,
|
|
|
|
| 43 |
self.app.prepare(ctx_id=0, det_size=(640, 640))
|
| 44 |
|
| 45 |
openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
|
| 46 |
+
depth_anything = DepthAnything.from_pretrained('LiheYoung/depth_anything_vitl14').to(device).eval()
|
| 47 |
+
|
| 48 |
+
transform = Compose([
|
| 49 |
+
Resize(
|
| 50 |
+
width=512,
|
| 51 |
+
height=512,
|
| 52 |
+
resize_target=False,
|
| 53 |
+
keep_aspect_ratio=True,
|
| 54 |
+
ensure_multiple_of=14,
|
| 55 |
+
resize_method='lower_bound',
|
| 56 |
+
image_interpolation_method=cv2.INTER_CUBIC,
|
| 57 |
+
),
|
| 58 |
+
NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
|
| 59 |
+
PrepareForNet(),
|
| 60 |
+
])
|
| 61 |
|
| 62 |
face_adapter = f"/repository/checkpoints/ip-adapter.bin"
|
| 63 |
controlnet_path = f"/repository/checkpoints/ControlNetModel"
|
|
|
|
| 68 |
|
| 69 |
controlnet_pose_model = "thibaud/controlnet-openpose-sdxl-1.0"
|
| 70 |
controlnet_canny_model = "diffusers/controlnet-canny-sdxl-1.0"
|
| 71 |
+
controlnet_depth_model = "diffusers/controlnet-depth-sdxl-1.0-small"
|
| 72 |
|
| 73 |
controlnet_pose = ControlNetModel.from_pretrained(
|
| 74 |
controlnet_pose_model, torch_dtype=dtype
|
|
|
|
| 76 |
controlnet_canny = ControlNetModel.from_pretrained(
|
| 77 |
controlnet_canny_model, torch_dtype=dtype
|
| 78 |
).to(device)
|
| 79 |
+
controlnet_depth = ControlNetModel.from_pretrained(
|
| 80 |
+
controlnet_depth_model, torch_dtype=dtype
|
| 81 |
+
).to(device)
|
| 82 |
|
| 83 |
+
def get_depth_map(image):
|
| 84 |
|
| 85 |
+
image = np.array(image) / 255.0
|
| 86 |
|
| 87 |
+
h, w = image.shape[:2]
|
| 88 |
|
| 89 |
+
image = transform({'image': image})['image']
|
| 90 |
+
image = torch.from_numpy(image).unsqueeze(0).to("cuda")
|
| 91 |
|
| 92 |
+
with torch.no_grad():
|
| 93 |
+
depth = depth_anything(image)
|
| 94 |
|
| 95 |
+
depth = F.interpolate(depth[None], (h, w), mode='bilinear', align_corners=False)[0, 0]
|
| 96 |
+
depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0
|
| 97 |
|
| 98 |
+
depth = depth.cpu().numpy().astype(np.uint8)
|
| 99 |
|
| 100 |
+
depth_image = Image.fromarray(depth)
|
| 101 |
|
| 102 |
+
return depth_image
|
| 103 |
|
| 104 |
def get_canny_image(image, t1=100, t2=200):
|
| 105 |
image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
|
|
|
|
| 109 |
self.controlnet_map = {
|
| 110 |
"pose": controlnet_pose,
|
| 111 |
"canny": controlnet_canny,
|
| 112 |
+
"depth": controlnet_depth,
|
| 113 |
}
|
| 114 |
|
| 115 |
self.controlnet_map_fn = {
|
| 116 |
"pose": openpose,
|
| 117 |
"canny": get_canny_image,
|
| 118 |
+
"depth": get_depth_map,
|
| 119 |
}
|
| 120 |
|
| 121 |
pretrained_model_name_or_path = "wangqixun/YamerMIX_v8"
|
|
|
|
| 142 |
self.pipe.unet.to("cuda")
|
| 143 |
|
| 144 |
# if we need more parameters
|
| 145 |
+
scheduler_class_name = "EulerDiscreteScheduler"
|
| 146 |
+
add_kwargs = {}
|
| 147 |
+
scheduler = getattr(diffusers, scheduler_class_name)
|
| 148 |
+
self.pipe.scheduler = scheduler.from_config(self.pipe.scheduler.config, **add_kwargs)
|
| 149 |
|
| 150 |
identitynet_strength_ratio = 0.8
|
| 151 |
|
|
|
|
| 153 |
canny_strength = 0.3
|
| 154 |
depth_strength = 0.5
|
| 155 |
|
| 156 |
+
# self.my_controlnet_selection = ["pose", "canny"]
|
| 157 |
+
self.my_controlnet_selection = ["pose", "canny", "depth"]
|
| 158 |
|
| 159 |
controlnet_scales = {
|
| 160 |
"pose": pose_strength,
|