manhdo's picture
Clean code and fix out of boundary prediction box
aef4429
Raw History Blame Contribute Delete
8.74 kB
import numpy as np
import cv2
from .general import resize_img
def detect_face_pose_informations_from_image(image, face_detection, face_mesh, img_size, pad_infos,
img_size_no_pad, head_pose_info, get_descending_order=True):
"""
Input:
`image` (np.ndarray): Image for face landmarks detection
`face_detection` (mp.solutions.face_detection): face detection model
`face_mesh` (mp.solutions.face_mesh): landmarks detection model
`img_size` (int, int): shape of the image (width, height)
`pad_infos` (int,int,int,int): number of padding each corner (top,bottom,left,right)
`img_size_no_pad` (int, int): new image size but no padding
`head_pose_info` (dict): head pose configs dictionary
Return:
`face_pose_infos` (list(list[float])): pose info for each face (x,y,z)
`face_direction_infos` (list[str]): direction for each face
"""
img_w, img_h = img_size
pad_top, pad_bot, pad_left, pad_right = pad_infos
chosen_rectangle_pos_list = []
face_detection_infos = []
face_coordinate_infos = []
face_pose_infos = []
face_direction_infos = []
face_position_infos = []
face_area_infos = []
# Get the result
detection_results = face_detection.process(image)
if detection_results.detections:
for _, detection in enumerate(detection_results.detections):
bbox_info = detection.location_data.relative_bounding_box
xmin = bbox_info.xmin
ymin = bbox_info.ymin
width = bbox_info.width
height = bbox_info.height
# Check area
width_ori = (width * img_w) / (img_w - pad_top - pad_bot)
height_ori = (height * img_h) / (img_h - pad_left - pad_right)
area = width_ori * height_ori
if area < head_pose_info['FACE_OPTIONS']['MIN_FACE_RATIO_OVER_IMAGE']:
continue
face_area_infos.append(area)
face_sample_pad = head_pose_info['FACE_SAMPLE_PAD']
x_center = xmin + width / 2
y_center = ymin + height / 2
face_detection_infos.append(detection)
x_center_int = int(x_center * img_w)
y_center_int = int(y_center * img_h)
x_center_int_ori = x_center_int - pad_left # x center according to original size
y_center_int_ori = y_center_int - pad_top # y center according to original size
x_center_ori = min(max(x_center_int_ori / img_size_no_pad[1], 0), 1)
y_center_ori = min(max(y_center_int_ori / img_size_no_pad[0], 0), 1)
xmax = int((xmin + width) * img_w)
ymax = int((ymin + height) * img_h)
xmin = int(xmin * img_w)
ymin = int(ymin * img_h)
face_coordinate_infos.append((xmin,ymin,xmax,ymax))
image_pad = cv2.copyMakeBorder(image.copy(), img_h, img_h, img_w, img_w, cv2.BORDER_CONSTANT, value=(0, 0, 0))
image_face_ori = image_pad[ymin + img_h - face_sample_pad:ymax + img_h + face_sample_pad,
xmin + img_w - face_sample_pad: xmax + img_h + face_sample_pad]
face_position = 'MIDDLE'
for head_pose_pos_name, head_pose_pos in head_pose_info['POSITION'].items():
if (head_pose_pos[0][0] < x_center_ori and x_center_ori <= head_pose_pos[1][0]) and \
(head_pose_pos[0][1] < y_center_ori and y_center_ori <= head_pose_pos[1][1]):
face_position = head_pose_pos_name
chosen_rectangle_pos_list.append(head_pose_pos)
break
face_position_infos.append(face_position)
face_pose_infos_for_cur_face, face_direction_infos_for_cur_face = \
detect_landmarks_from_image(image_face_ori, face_mesh, img_size, head_pose_info, get_one_landmark=True)
face_pose_infos.extend(face_pose_infos_for_cur_face)
face_direction_infos.extend(face_direction_infos_for_cur_face)
if get_descending_order: # according to face area
face_orders = np.argsort(face_area_infos)[::-1]
face_detection_infos = list(np.array(face_detection_infos)[face_orders])
face_direction_infos = list(np.array(face_direction_infos)[face_orders])
face_position_infos = list(np.array(face_position_infos)[face_orders])
face_coordinate_infos = list(np.array(face_coordinate_infos)[face_orders])
face_pose_infos = list(np.array(face_pose_infos)[face_orders])
face_area_infos = list(np.array(face_area_infos)[face_orders])
chosen_rectangle_pos_list = [chosen_rectangle_pos for _, chosen_rectangle_pos in sorted(zip(face_orders, chosen_rectangle_pos_list))]
return face_detection_infos, face_direction_infos, face_position_infos, face_coordinate_infos, face_pose_infos, face_area_infos, chosen_rectangle_pos_list
def detect_landmarks_from_image(image, face_mesh, img_size, head_pose_info, get_one_landmark=False):
"""
Input:
`image` (np.ndarray): Image for face landmarks detection
`img_size` (int, int): shape of the image (width, height)
`face_mesh` (mp.solutions.face_mesh): landmarks detection model
`get_one_landmark` (bool): In case you just want predict one landmarks for one face, `True` for return one landmarks infos, `False` otherwise
`head_pose_info` (dict): head pose configs dictionary
Return:
`face_pose_infos` (list(list[float])): pose info for each face (x,y,z)
`face_direction_infos` (list[str]): direction for each face
"""
image_resize = resize_img(image, img_size)
results = face_mesh.process(image_resize)
# Multi landmarks informations from landmarks detection model
multi_face_landmarks = results.multi_face_landmarks
if not multi_face_landmarks:
return [(0, 0, 0)], ['NO FACE']
else:
face_pose_infos, face_direction_infos = [], []
img_w, img_h = img_size
face_3d = []
face_2d = []
for face_landmarks in multi_face_landmarks:
for idx, lm in enumerate(face_landmarks.landmark):
if idx == 33 or idx == 263 or idx == 1 or idx == 61 or idx == 291 or idx == 199:
if idx == 1:
nose_2d = (lm.x * img_w, lm.y * img_h)
nose_3d = (lm.x * img_w, lm.y * img_h, lm.z * 3000)
x, y = int(lm.x * img_w), int(lm.y * img_h)
# Get the 2D Coordinates
face_2d.append([x, y])
# Get the 3D Coordinates
face_3d.append([x, y, lm.z])
# Convert to Numpy array
face_2d = np.array(face_2d, dtype=np.float64)
face_3d = np.array(face_3d, dtype=np.float64)
# The camera matrix
focal_length = 1 * img_w
cam_matrix = np.array([ [focal_length, 0, img_h/2],
[0, focal_length, img_w/2],
[0, 0, 1]])
# The distortion parameters
dist_matrix = np.zeros((4, 1), dtype=np.float64)
# Solve PnP
success, rot_vec, trans_vec = cv2.solvePnP(face_3d, face_2d, cam_matrix,dist_matrix)
# Get rotational matrix
rmat, jac = cv2.Rodrigues(rot_vec)
# Get angles
angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat)
# Get the y rotation degree
x = angles[0] * 360
y = angles[1] * 360
z = angles[2] * 360
face_pose_infos.append((x,y,z))
# See where the user's head tilting
face_direction = 'NOT FRONT'
for head_pose_direction_name, head_pose_direction in head_pose_info['DIRECTION'].items():
if (head_pose_direction[0][0] < x and x <= head_pose_direction[1][0]) and \
(head_pose_direction[0][1] < y and y <= head_pose_direction[1][1]):
face_direction = head_pose_direction_name
break
face_direction_infos.append(face_direction)
# Display the nose direction
nose_3d_projection, jacobian = cv2.projectPoints(nose_3d, rot_vec, trans_vec, cam_matrix, dist_matrix)
p1 = (int(nose_2d[0]), int(nose_2d[1]))
p2 = (int(nose_2d[0] + y*10), int(nose_2d[1] - x*10))
# cv2.line(image_face, p1, p2, (255, 0, 0), 3)
if get_one_landmark:
break
return face_pose_infos, face_direction_infos