Jacobmadwed commited on
Commit
3fd28a6
·
verified ·
1 Parent(s): 4e4047b

Update handler.py

Browse files
Files changed (1) hide show
  1. handler.py +15 -14
handler.py CHANGED
@@ -20,7 +20,7 @@ import base64
20
  import io
21
  import json
22
  from transformers import CLIPProcessor, CLIPModel
23
- from keras.models import load_model
24
 
25
  # global variable
26
  device = "cuda" if torch.cuda.is_available() else "cpu"
@@ -58,11 +58,11 @@ class EndpointHandler:
58
  controlnet_model = hf_hub_download(repo_id="InstantX/InstantID", filename="ControlNetModel/diffusion_pytorch_model.safetensors", local_dir=os.path.join(model_dir, "checkpoints"))
59
  face_adapter = hf_hub_download(repo_id="InstantX/InstantID", filename="ip-adapter.bin", local_dir=os.path.join(model_dir, "checkpoints"))
60
 
61
- # Load the Keras-OpenFace model
62
- openface_model_path = os.path.join(model_dir, "models", "nn4.small2.v1.h5")
63
- if not os.path.exists(openface_model_path):
64
- print(f"Model path {openface_model_path} does not exist. Please ensure the model is available.")
65
- self.openface_model = load_model(openface_model_path)
66
 
67
  openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
68
 
@@ -147,19 +147,20 @@ class EndpointHandler:
147
  return nsfw_prob > 0.8 # Adjusted threshold for NSFW detection
148
 
149
  def preprocess(self, image):
150
- # Preprocess the image for Keras-OpenFace
151
- image = cv2.resize(image, (96, 96))
152
- image = image[..., ::-1] # BGR to RGB
153
  image = np.transpose(image, (2, 0, 1))
154
- image = image[np.newaxis, :, :, :].astype(np.float32) / 255.0 # Normalize to [0, 1]
155
  return image
156
 
157
  def get_face_embedding(self, image):
158
  # Preprocess the image
159
  image = self.preprocess(image)
160
 
161
- # Run the Keras-OpenFace model to get the face embedding
162
- embedding = self.openface_model.predict(image)
 
163
 
164
  return embedding
165
 
@@ -249,7 +250,7 @@ class EndpointHandler:
249
  face_image_cv2 = convert_from_image_to_cv2(face_image)
250
  height, width, _ = face_image_cv2.shape
251
 
252
- # Extract face features using the Keras-OpenFace model
253
  face_emb = self.get_face_embedding(face_image_cv2)
254
 
255
  face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info["kps"])
@@ -259,7 +260,7 @@ class EndpointHandler:
259
  img_controlnet = pose_image
260
  pose_image_cv2 = convert_from_image_to_cv2(pose_image)
261
 
262
- # Extract face features from pose image using the Keras-OpenFace model
263
  face_emb = self.get_face_embedding(pose_image_cv2)
264
  face_kps = draw_kps(pose_image, face_info["kps"])
265
 
 
20
  import io
21
  import json
22
  from transformers import CLIPProcessor, CLIPModel
23
+ import onnxruntime as ort
24
 
25
  # global variable
26
  device = "cuda" if torch.cuda.is_available() else "cpu"
 
58
  controlnet_model = hf_hub_download(repo_id="InstantX/InstantID", filename="ControlNetModel/diffusion_pytorch_model.safetensors", local_dir=os.path.join(model_dir, "checkpoints"))
59
  face_adapter = hf_hub_download(repo_id="InstantX/InstantID", filename="ip-adapter.bin", local_dir=os.path.join(model_dir, "checkpoints"))
60
 
61
+ # Load the ONNX model
62
+ onnx_model_path = os.path.join(model_dir, "models", "version-RFB-320.onnx")
63
+ if not os.path.exists(onnx_model_path):
64
+ print(f"Model path {onnx_model_path} does not exist. Please ensure the model is available.")
65
+ self.ort_session = ort.InferenceSession(onnx_model_path)
66
 
67
  openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
68
 
 
147
  return nsfw_prob > 0.8 # Adjusted threshold for NSFW detection
148
 
149
  def preprocess(self, image):
150
+ # Preprocess the image for ONNX model
151
+ image = cv2.resize(image, (320, 240)) # Adjust based on model input size
152
+ image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
153
  image = np.transpose(image, (2, 0, 1))
154
+ image = image[np.newaxis, :, :, :].astype(np.float32) / 127.5 - 1.0 # Normalize to [-1, 1]
155
  return image
156
 
157
  def get_face_embedding(self, image):
158
  # Preprocess the image
159
  image = self.preprocess(image)
160
 
161
+ # Run the ONNX model to get the face embedding
162
+ input_name = self.ort_session.get_inputs()[0].name
163
+ embedding = self.ort_session.run(None, {input_name: image})[0]
164
 
165
  return embedding
166
 
 
250
  face_image_cv2 = convert_from_image_to_cv2(face_image)
251
  height, width, _ = face_image_cv2.shape
252
 
253
+ # Extract face features using the ONNX model
254
  face_emb = self.get_face_embedding(face_image_cv2)
255
 
256
  face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info["kps"])
 
260
  img_controlnet = pose_image
261
  pose_image_cv2 = convert_from_image_to_cv2(pose_image)
262
 
263
+ # Extract face features from pose image using the ONNX model
264
  face_emb = self.get_face_embedding(pose_image_cv2)
265
  face_kps = draw_kps(pose_image, face_info["kps"])
266