Spaces:
Runtime error
Runtime error
File size: 8,740 Bytes
aef4429 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 | import numpy as np
import cv2
from .general import resize_img
def detect_face_pose_informations_from_image(image, face_detection, face_mesh, img_size, pad_infos,
img_size_no_pad, head_pose_info, get_descending_order=True):
"""
Input:
`image` (np.ndarray): Image for face landmarks detection
`face_detection` (mp.solutions.face_detection): face detection model
`face_mesh` (mp.solutions.face_mesh): landmarks detection model
`img_size` (int, int): shape of the image (width, height)
`pad_infos` (int,int,int,int): number of padding each corner (top,bottom,left,right)
`img_size_no_pad` (int, int): new image size but no padding
`head_pose_info` (dict): head pose configs dictionary
Return:
`face_pose_infos` (list(list[float])): pose info for each face (x,y,z)
`face_direction_infos` (list[str]): direction for each face
"""
img_w, img_h = img_size
pad_top, pad_bot, pad_left, pad_right = pad_infos
chosen_rectangle_pos_list = []
face_detection_infos = []
face_coordinate_infos = []
face_pose_infos = []
face_direction_infos = []
face_position_infos = []
face_area_infos = []
# Get the result
detection_results = face_detection.process(image)
if detection_results.detections:
for _, detection in enumerate(detection_results.detections):
bbox_info = detection.location_data.relative_bounding_box
xmin = bbox_info.xmin
ymin = bbox_info.ymin
width = bbox_info.width
height = bbox_info.height
# Check area
width_ori = (width * img_w) / (img_w - pad_top - pad_bot)
height_ori = (height * img_h) / (img_h - pad_left - pad_right)
area = width_ori * height_ori
if area < head_pose_info['FACE_OPTIONS']['MIN_FACE_RATIO_OVER_IMAGE']:
continue
face_area_infos.append(area)
face_sample_pad = head_pose_info['FACE_SAMPLE_PAD']
x_center = xmin + width / 2
y_center = ymin + height / 2
face_detection_infos.append(detection)
x_center_int = int(x_center * img_w)
y_center_int = int(y_center * img_h)
x_center_int_ori = x_center_int - pad_left # x center according to original size
y_center_int_ori = y_center_int - pad_top # y center according to original size
x_center_ori = min(max(x_center_int_ori / img_size_no_pad[1], 0), 1)
y_center_ori = min(max(y_center_int_ori / img_size_no_pad[0], 0), 1)
xmax = int((xmin + width) * img_w)
ymax = int((ymin + height) * img_h)
xmin = int(xmin * img_w)
ymin = int(ymin * img_h)
face_coordinate_infos.append((xmin,ymin,xmax,ymax))
image_pad = cv2.copyMakeBorder(image.copy(), img_h, img_h, img_w, img_w, cv2.BORDER_CONSTANT, value=(0, 0, 0))
image_face_ori = image_pad[ymin + img_h - face_sample_pad:ymax + img_h + face_sample_pad,
xmin + img_w - face_sample_pad: xmax + img_h + face_sample_pad]
face_position = 'MIDDLE'
for head_pose_pos_name, head_pose_pos in head_pose_info['POSITION'].items():
if (head_pose_pos[0][0] < x_center_ori and x_center_ori <= head_pose_pos[1][0]) and \
(head_pose_pos[0][1] < y_center_ori and y_center_ori <= head_pose_pos[1][1]):
face_position = head_pose_pos_name
chosen_rectangle_pos_list.append(head_pose_pos)
break
face_position_infos.append(face_position)
face_pose_infos_for_cur_face, face_direction_infos_for_cur_face = \
detect_landmarks_from_image(image_face_ori, face_mesh, img_size, head_pose_info, get_one_landmark=True)
face_pose_infos.extend(face_pose_infos_for_cur_face)
face_direction_infos.extend(face_direction_infos_for_cur_face)
if get_descending_order: # according to face area
face_orders = np.argsort(face_area_infos)[::-1]
face_detection_infos = list(np.array(face_detection_infos)[face_orders])
face_direction_infos = list(np.array(face_direction_infos)[face_orders])
face_position_infos = list(np.array(face_position_infos)[face_orders])
face_coordinate_infos = list(np.array(face_coordinate_infos)[face_orders])
face_pose_infos = list(np.array(face_pose_infos)[face_orders])
face_area_infos = list(np.array(face_area_infos)[face_orders])
chosen_rectangle_pos_list = [chosen_rectangle_pos for _, chosen_rectangle_pos in sorted(zip(face_orders, chosen_rectangle_pos_list))]
return face_detection_infos, face_direction_infos, face_position_infos, face_coordinate_infos, face_pose_infos, face_area_infos, chosen_rectangle_pos_list
def detect_landmarks_from_image(image, face_mesh, img_size, head_pose_info, get_one_landmark=False):
"""
Input:
`image` (np.ndarray): Image for face landmarks detection
`img_size` (int, int): shape of the image (width, height)
`face_mesh` (mp.solutions.face_mesh): landmarks detection model
`get_one_landmark` (bool): In case you just want predict one landmarks for one face, `True` for return one landmarks infos, `False` otherwise
`head_pose_info` (dict): head pose configs dictionary
Return:
`face_pose_infos` (list(list[float])): pose info for each face (x,y,z)
`face_direction_infos` (list[str]): direction for each face
"""
image_resize = resize_img(image, img_size)
results = face_mesh.process(image_resize)
# Multi landmarks informations from landmarks detection model
multi_face_landmarks = results.multi_face_landmarks
if not multi_face_landmarks:
return [(0, 0, 0)], ['NO FACE']
else:
face_pose_infos, face_direction_infos = [], []
img_w, img_h = img_size
face_3d = []
face_2d = []
for face_landmarks in multi_face_landmarks:
for idx, lm in enumerate(face_landmarks.landmark):
if idx == 33 or idx == 263 or idx == 1 or idx == 61 or idx == 291 or idx == 199:
if idx == 1:
nose_2d = (lm.x * img_w, lm.y * img_h)
nose_3d = (lm.x * img_w, lm.y * img_h, lm.z * 3000)
x, y = int(lm.x * img_w), int(lm.y * img_h)
# Get the 2D Coordinates
face_2d.append([x, y])
# Get the 3D Coordinates
face_3d.append([x, y, lm.z])
# Convert to Numpy array
face_2d = np.array(face_2d, dtype=np.float64)
face_3d = np.array(face_3d, dtype=np.float64)
# The camera matrix
focal_length = 1 * img_w
cam_matrix = np.array([ [focal_length, 0, img_h/2],
[0, focal_length, img_w/2],
[0, 0, 1]])
# The distortion parameters
dist_matrix = np.zeros((4, 1), dtype=np.float64)
# Solve PnP
success, rot_vec, trans_vec = cv2.solvePnP(face_3d, face_2d, cam_matrix,dist_matrix)
# Get rotational matrix
rmat, jac = cv2.Rodrigues(rot_vec)
# Get angles
angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat)
# Get the y rotation degree
x = angles[0] * 360
y = angles[1] * 360
z = angles[2] * 360
face_pose_infos.append((x,y,z))
# See where the user's head tilting
face_direction = 'NOT FRONT'
for head_pose_direction_name, head_pose_direction in head_pose_info['DIRECTION'].items():
if (head_pose_direction[0][0] < x and x <= head_pose_direction[1][0]) and \
(head_pose_direction[0][1] < y and y <= head_pose_direction[1][1]):
face_direction = head_pose_direction_name
break
face_direction_infos.append(face_direction)
# Display the nose direction
nose_3d_projection, jacobian = cv2.projectPoints(nose_3d, rot_vec, trans_vec, cam_matrix, dist_matrix)
p1 = (int(nose_2d[0]), int(nose_2d[1]))
p2 = (int(nose_2d[0] + y*10), int(nose_2d[1] - x*10))
# cv2.line(image_face, p1, p2, (255, 0, 0), 3)
if get_one_landmark:
break
return face_pose_infos, face_direction_infos |