import numpy as np import cv2 from .general import resize_img def detect_face_pose_informations_from_image(image, face_detection, face_mesh, img_size, pad_infos, img_size_no_pad, head_pose_info, get_descending_order=True): """ Input: `image` (np.ndarray): Image for face landmarks detection `face_detection` (mp.solutions.face_detection): face detection model `face_mesh` (mp.solutions.face_mesh): landmarks detection model `img_size` (int, int): shape of the image (width, height) `pad_infos` (int,int,int,int): number of padding each corner (top,bottom,left,right) `img_size_no_pad` (int, int): new image size but no padding `head_pose_info` (dict): head pose configs dictionary Return: `face_pose_infos` (list(list[float])): pose info for each face (x,y,z) `face_direction_infos` (list[str]): direction for each face """ img_w, img_h = img_size pad_top, pad_bot, pad_left, pad_right = pad_infos chosen_rectangle_pos_list = [] face_detection_infos = [] face_coordinate_infos = [] face_pose_infos = [] face_direction_infos = [] face_position_infos = [] face_area_infos = [] # Get the result detection_results = face_detection.process(image) if detection_results.detections: for _, detection in enumerate(detection_results.detections): bbox_info = detection.location_data.relative_bounding_box xmin = bbox_info.xmin ymin = bbox_info.ymin width = bbox_info.width height = bbox_info.height # Check area width_ori = (width * img_w) / (img_w - pad_top - pad_bot) height_ori = (height * img_h) / (img_h - pad_left - pad_right) area = width_ori * height_ori if area < head_pose_info['FACE_OPTIONS']['MIN_FACE_RATIO_OVER_IMAGE']: continue face_area_infos.append(area) face_sample_pad = head_pose_info['FACE_SAMPLE_PAD'] x_center = xmin + width / 2 y_center = ymin + height / 2 face_detection_infos.append(detection) x_center_int = int(x_center * img_w) y_center_int = int(y_center * img_h) x_center_int_ori = x_center_int - pad_left # x center according to original size y_center_int_ori = y_center_int - pad_top # y center according to original size x_center_ori = min(max(x_center_int_ori / img_size_no_pad[1], 0), 1) y_center_ori = min(max(y_center_int_ori / img_size_no_pad[0], 0), 1) xmax = int((xmin + width) * img_w) ymax = int((ymin + height) * img_h) xmin = int(xmin * img_w) ymin = int(ymin * img_h) face_coordinate_infos.append((xmin,ymin,xmax,ymax)) image_pad = cv2.copyMakeBorder(image.copy(), img_h, img_h, img_w, img_w, cv2.BORDER_CONSTANT, value=(0, 0, 0)) image_face_ori = image_pad[ymin + img_h - face_sample_pad:ymax + img_h + face_sample_pad, xmin + img_w - face_sample_pad: xmax + img_h + face_sample_pad] face_position = 'MIDDLE' for head_pose_pos_name, head_pose_pos in head_pose_info['POSITION'].items(): if (head_pose_pos[0][0] < x_center_ori and x_center_ori <= head_pose_pos[1][0]) and \ (head_pose_pos[0][1] < y_center_ori and y_center_ori <= head_pose_pos[1][1]): face_position = head_pose_pos_name chosen_rectangle_pos_list.append(head_pose_pos) break face_position_infos.append(face_position) face_pose_infos_for_cur_face, face_direction_infos_for_cur_face = \ detect_landmarks_from_image(image_face_ori, face_mesh, img_size, head_pose_info, get_one_landmark=True) face_pose_infos.extend(face_pose_infos_for_cur_face) face_direction_infos.extend(face_direction_infos_for_cur_face) if get_descending_order: # according to face area face_orders = np.argsort(face_area_infos)[::-1] face_detection_infos = list(np.array(face_detection_infos)[face_orders]) face_direction_infos = list(np.array(face_direction_infos)[face_orders]) face_position_infos = list(np.array(face_position_infos)[face_orders]) face_coordinate_infos = list(np.array(face_coordinate_infos)[face_orders]) face_pose_infos = list(np.array(face_pose_infos)[face_orders]) face_area_infos = list(np.array(face_area_infos)[face_orders]) chosen_rectangle_pos_list = [chosen_rectangle_pos for _, chosen_rectangle_pos in sorted(zip(face_orders, chosen_rectangle_pos_list))] return face_detection_infos, face_direction_infos, face_position_infos, face_coordinate_infos, face_pose_infos, face_area_infos, chosen_rectangle_pos_list def detect_landmarks_from_image(image, face_mesh, img_size, head_pose_info, get_one_landmark=False): """ Input: `image` (np.ndarray): Image for face landmarks detection `img_size` (int, int): shape of the image (width, height) `face_mesh` (mp.solutions.face_mesh): landmarks detection model `get_one_landmark` (bool): In case you just want predict one landmarks for one face, `True` for return one landmarks infos, `False` otherwise `head_pose_info` (dict): head pose configs dictionary Return: `face_pose_infos` (list(list[float])): pose info for each face (x,y,z) `face_direction_infos` (list[str]): direction for each face """ image_resize = resize_img(image, img_size) results = face_mesh.process(image_resize) # Multi landmarks informations from landmarks detection model multi_face_landmarks = results.multi_face_landmarks if not multi_face_landmarks: return [(0, 0, 0)], ['NO FACE'] else: face_pose_infos, face_direction_infos = [], [] img_w, img_h = img_size face_3d = [] face_2d = [] for face_landmarks in multi_face_landmarks: for idx, lm in enumerate(face_landmarks.landmark): if idx == 33 or idx == 263 or idx == 1 or idx == 61 or idx == 291 or idx == 199: if idx == 1: nose_2d = (lm.x * img_w, lm.y * img_h) nose_3d = (lm.x * img_w, lm.y * img_h, lm.z * 3000) x, y = int(lm.x * img_w), int(lm.y * img_h) # Get the 2D Coordinates face_2d.append([x, y]) # Get the 3D Coordinates face_3d.append([x, y, lm.z]) # Convert to Numpy array face_2d = np.array(face_2d, dtype=np.float64) face_3d = np.array(face_3d, dtype=np.float64) # The camera matrix focal_length = 1 * img_w cam_matrix = np.array([ [focal_length, 0, img_h/2], [0, focal_length, img_w/2], [0, 0, 1]]) # The distortion parameters dist_matrix = np.zeros((4, 1), dtype=np.float64) # Solve PnP success, rot_vec, trans_vec = cv2.solvePnP(face_3d, face_2d, cam_matrix,dist_matrix) # Get rotational matrix rmat, jac = cv2.Rodrigues(rot_vec) # Get angles angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat) # Get the y rotation degree x = angles[0] * 360 y = angles[1] * 360 z = angles[2] * 360 face_pose_infos.append((x,y,z)) # See where the user's head tilting face_direction = 'NOT FRONT' for head_pose_direction_name, head_pose_direction in head_pose_info['DIRECTION'].items(): if (head_pose_direction[0][0] < x and x <= head_pose_direction[1][0]) and \ (head_pose_direction[0][1] < y and y <= head_pose_direction[1][1]): face_direction = head_pose_direction_name break face_direction_infos.append(face_direction) # Display the nose direction nose_3d_projection, jacobian = cv2.projectPoints(nose_3d, rot_vec, trans_vec, cam_matrix, dist_matrix) p1 = (int(nose_2d[0]), int(nose_2d[1])) p2 = (int(nose_2d[0] + y*10), int(nose_2d[1] - x*10)) # cv2.line(image_face, p1, p2, (255, 0, 0), 3) if get_one_landmark: break return face_pose_infos, face_direction_infos