yamildiego commited on
Commit
c9fb62c
·
1 Parent(s): 4073f23

added depth

Browse files
Files changed (1) hide show
  1. handler.py +39 -39
handler.py CHANGED
@@ -43,21 +43,21 @@ class EndpointHandler():
43
  self.app.prepare(ctx_id=0, det_size=(640, 640))
44
 
45
  openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
46
- # depth_anything = DepthAnything.from_pretrained('LiheYoung/depth_anything_vitl14').to(device).eval()
47
-
48
- # transform = Compose([
49
- # Resize(
50
- # width=512,
51
- # height=512,
52
- # resize_target=False,
53
- # keep_aspect_ratio=True,
54
- # ensure_multiple_of=14,
55
- # resize_method='lower_bound',
56
- # image_interpolation_method=cv2.INTER_CUBIC,
57
- # ),
58
- # NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
59
- # PrepareForNet(),
60
- # ])
61
 
62
  face_adapter = f"/repository/checkpoints/ip-adapter.bin"
63
  controlnet_path = f"/repository/checkpoints/ControlNetModel"
@@ -68,7 +68,7 @@ class EndpointHandler():
68
 
69
  controlnet_pose_model = "thibaud/controlnet-openpose-sdxl-1.0"
70
  controlnet_canny_model = "diffusers/controlnet-canny-sdxl-1.0"
71
- # controlnet_depth_model = "diffusers/controlnet-depth-sdxl-1.0-small"
72
 
73
  controlnet_pose = ControlNetModel.from_pretrained(
74
  controlnet_pose_model, torch_dtype=dtype
@@ -76,30 +76,30 @@ class EndpointHandler():
76
  controlnet_canny = ControlNetModel.from_pretrained(
77
  controlnet_canny_model, torch_dtype=dtype
78
  ).to(device)
79
- # controlnet_depth = ControlNetModel.from_pretrained(
80
- # controlnet_depth_model, torch_dtype=dtype
81
- # ).to(device)
82
 
83
- # def get_depth_map(image):
84
 
85
- # image = np.array(image) / 255.0
86
 
87
- # h, w = image.shape[:2]
88
 
89
- # image = transform({'image': image})['image']
90
- # image = torch.from_numpy(image).unsqueeze(0).to("cuda")
91
 
92
- # with torch.no_grad():
93
- # depth = depth_anything(image)
94
 
95
- # depth = F.interpolate(depth[None], (h, w), mode='bilinear', align_corners=False)[0, 0]
96
- # depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0
97
 
98
- # depth = depth.cpu().numpy().astype(np.uint8)
99
 
100
- # depth_image = Image.fromarray(depth)
101
 
102
- # return depth_image
103
 
104
  def get_canny_image(image, t1=100, t2=200):
105
  image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
@@ -109,13 +109,13 @@ class EndpointHandler():
109
  self.controlnet_map = {
110
  "pose": controlnet_pose,
111
  "canny": controlnet_canny,
112
- # "depth": controlnet_depth,
113
  }
114
 
115
  self.controlnet_map_fn = {
116
  "pose": openpose,
117
  "canny": get_canny_image,
118
- # "depth": get_depth_map,
119
  }
120
 
121
  pretrained_model_name_or_path = "wangqixun/YamerMIX_v8"
@@ -142,10 +142,10 @@ class EndpointHandler():
142
  self.pipe.unet.to("cuda")
143
 
144
  # if we need more parameters
145
- # scheduler_class_name = "EulerDiscreteScheduler"
146
- # add_kwargs = {}
147
- # scheduler = getattr(diffusers, scheduler_class_name)
148
- # self.pipe.scheduler = scheduler.from_config(self.pipe.scheduler.config, **add_kwargs)
149
 
150
  identitynet_strength_ratio = 0.8
151
 
@@ -153,8 +153,8 @@ class EndpointHandler():
153
  canny_strength = 0.3
154
  depth_strength = 0.5
155
 
156
- self.my_controlnet_selection = ["pose", "canny"]
157
- # self.my_controlnet_selection = ["pose", "canny", "depth"]
158
 
159
  controlnet_scales = {
160
  "pose": pose_strength,
 
43
  self.app.prepare(ctx_id=0, det_size=(640, 640))
44
 
45
  openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
46
+ depth_anything = DepthAnything.from_pretrained('LiheYoung/depth_anything_vitl14').to(device).eval()
47
+
48
+ transform = Compose([
49
+ Resize(
50
+ width=512,
51
+ height=512,
52
+ resize_target=False,
53
+ keep_aspect_ratio=True,
54
+ ensure_multiple_of=14,
55
+ resize_method='lower_bound',
56
+ image_interpolation_method=cv2.INTER_CUBIC,
57
+ ),
58
+ NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
59
+ PrepareForNet(),
60
+ ])
61
 
62
  face_adapter = f"/repository/checkpoints/ip-adapter.bin"
63
  controlnet_path = f"/repository/checkpoints/ControlNetModel"
 
68
 
69
  controlnet_pose_model = "thibaud/controlnet-openpose-sdxl-1.0"
70
  controlnet_canny_model = "diffusers/controlnet-canny-sdxl-1.0"
71
+ controlnet_depth_model = "diffusers/controlnet-depth-sdxl-1.0-small"
72
 
73
  controlnet_pose = ControlNetModel.from_pretrained(
74
  controlnet_pose_model, torch_dtype=dtype
 
76
  controlnet_canny = ControlNetModel.from_pretrained(
77
  controlnet_canny_model, torch_dtype=dtype
78
  ).to(device)
79
+ controlnet_depth = ControlNetModel.from_pretrained(
80
+ controlnet_depth_model, torch_dtype=dtype
81
+ ).to(device)
82
 
83
+ def get_depth_map(image):
84
 
85
+ image = np.array(image) / 255.0
86
 
87
+ h, w = image.shape[:2]
88
 
89
+ image = transform({'image': image})['image']
90
+ image = torch.from_numpy(image).unsqueeze(0).to("cuda")
91
 
92
+ with torch.no_grad():
93
+ depth = depth_anything(image)
94
 
95
+ depth = F.interpolate(depth[None], (h, w), mode='bilinear', align_corners=False)[0, 0]
96
+ depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0
97
 
98
+ depth = depth.cpu().numpy().astype(np.uint8)
99
 
100
+ depth_image = Image.fromarray(depth)
101
 
102
+ return depth_image
103
 
104
  def get_canny_image(image, t1=100, t2=200):
105
  image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
 
109
  self.controlnet_map = {
110
  "pose": controlnet_pose,
111
  "canny": controlnet_canny,
112
+ "depth": controlnet_depth,
113
  }
114
 
115
  self.controlnet_map_fn = {
116
  "pose": openpose,
117
  "canny": get_canny_image,
118
+ "depth": get_depth_map,
119
  }
120
 
121
  pretrained_model_name_or_path = "wangqixun/YamerMIX_v8"
 
142
  self.pipe.unet.to("cuda")
143
 
144
  # if we need more parameters
145
+ scheduler_class_name = "EulerDiscreteScheduler"
146
+ add_kwargs = {}
147
+ scheduler = getattr(diffusers, scheduler_class_name)
148
+ self.pipe.scheduler = scheduler.from_config(self.pipe.scheduler.config, **add_kwargs)
149
 
150
  identitynet_strength_ratio = 0.8
151
 
 
153
  canny_strength = 0.3
154
  depth_strength = 0.5
155
 
156
+ # self.my_controlnet_selection = ["pose", "canny"]
157
+ self.my_controlnet_selection = ["pose", "canny", "depth"]
158
 
159
  controlnet_scales = {
160
  "pose": pose_strength,