Spaces:
Runtime error
Runtime error
Download utils/detection.py from manhdo/head_pose_estimation_tracking_app: direct link, hf CLI and curl.
- Browser
- Download file 8.74 kB
-
https://huggingface.co/spaces/manhdo/head_pose_estimation_tracking_app/resolve/main/utils/detection.py
- Command line
-
hf download hf://spaces/manhdo/head_pose_estimation_tracking_app/utils/detection.py
-
curl -L -o detection.py https://huggingface.co/spaces/manhdo/head_pose_estimation_tracking_app/resolve/main/utils/detection.py
8.74 kB
| import numpy as np | |
| import cv2 | |
| from .general import resize_img | |
| def detect_face_pose_informations_from_image(image, face_detection, face_mesh, img_size, pad_infos, | |
| img_size_no_pad, head_pose_info, get_descending_order=True): | |
| """ | |
| Input: | |
| `image` (np.ndarray): Image for face landmarks detection | |
| `face_detection` (mp.solutions.face_detection): face detection model | |
| `face_mesh` (mp.solutions.face_mesh): landmarks detection model | |
| `img_size` (int, int): shape of the image (width, height) | |
| `pad_infos` (int,int,int,int): number of padding each corner (top,bottom,left,right) | |
| `img_size_no_pad` (int, int): new image size but no padding | |
| `head_pose_info` (dict): head pose configs dictionary | |
| Return: | |
| `face_pose_infos` (list(list[float])): pose info for each face (x,y,z) | |
| `face_direction_infos` (list[str]): direction for each face | |
| """ | |
| img_w, img_h = img_size | |
| pad_top, pad_bot, pad_left, pad_right = pad_infos | |
| chosen_rectangle_pos_list = [] | |
| face_detection_infos = [] | |
| face_coordinate_infos = [] | |
| face_pose_infos = [] | |
| face_direction_infos = [] | |
| face_position_infos = [] | |
| face_area_infos = [] | |
| # Get the result | |
| detection_results = face_detection.process(image) | |
| if detection_results.detections: | |
| for _, detection in enumerate(detection_results.detections): | |
| bbox_info = detection.location_data.relative_bounding_box | |
| xmin = bbox_info.xmin | |
| ymin = bbox_info.ymin | |
| width = bbox_info.width | |
| height = bbox_info.height | |
| # Check area | |
| width_ori = (width * img_w) / (img_w - pad_top - pad_bot) | |
| height_ori = (height * img_h) / (img_h - pad_left - pad_right) | |
| area = width_ori * height_ori | |
| if area < head_pose_info['FACE_OPTIONS']['MIN_FACE_RATIO_OVER_IMAGE']: | |
| continue | |
| face_area_infos.append(area) | |
| face_sample_pad = head_pose_info['FACE_SAMPLE_PAD'] | |
| x_center = xmin + width / 2 | |
| y_center = ymin + height / 2 | |
| face_detection_infos.append(detection) | |
| x_center_int = int(x_center * img_w) | |
| y_center_int = int(y_center * img_h) | |
| x_center_int_ori = x_center_int - pad_left # x center according to original size | |
| y_center_int_ori = y_center_int - pad_top # y center according to original size | |
| x_center_ori = min(max(x_center_int_ori / img_size_no_pad[1], 0), 1) | |
| y_center_ori = min(max(y_center_int_ori / img_size_no_pad[0], 0), 1) | |
| xmax = int((xmin + width) * img_w) | |
| ymax = int((ymin + height) * img_h) | |
| xmin = int(xmin * img_w) | |
| ymin = int(ymin * img_h) | |
| face_coordinate_infos.append((xmin,ymin,xmax,ymax)) | |
| image_pad = cv2.copyMakeBorder(image.copy(), img_h, img_h, img_w, img_w, cv2.BORDER_CONSTANT, value=(0, 0, 0)) | |
| image_face_ori = image_pad[ymin + img_h - face_sample_pad:ymax + img_h + face_sample_pad, | |
| xmin + img_w - face_sample_pad: xmax + img_h + face_sample_pad] | |
| face_position = 'MIDDLE' | |
| for head_pose_pos_name, head_pose_pos in head_pose_info['POSITION'].items(): | |
| if (head_pose_pos[0][0] < x_center_ori and x_center_ori <= head_pose_pos[1][0]) and \ | |
| (head_pose_pos[0][1] < y_center_ori and y_center_ori <= head_pose_pos[1][1]): | |
| face_position = head_pose_pos_name | |
| chosen_rectangle_pos_list.append(head_pose_pos) | |
| break | |
| face_position_infos.append(face_position) | |
| face_pose_infos_for_cur_face, face_direction_infos_for_cur_face = \ | |
| detect_landmarks_from_image(image_face_ori, face_mesh, img_size, head_pose_info, get_one_landmark=True) | |
| face_pose_infos.extend(face_pose_infos_for_cur_face) | |
| face_direction_infos.extend(face_direction_infos_for_cur_face) | |
| if get_descending_order: # according to face area | |
| face_orders = np.argsort(face_area_infos)[::-1] | |
| face_detection_infos = list(np.array(face_detection_infos)[face_orders]) | |
| face_direction_infos = list(np.array(face_direction_infos)[face_orders]) | |
| face_position_infos = list(np.array(face_position_infos)[face_orders]) | |
| face_coordinate_infos = list(np.array(face_coordinate_infos)[face_orders]) | |
| face_pose_infos = list(np.array(face_pose_infos)[face_orders]) | |
| face_area_infos = list(np.array(face_area_infos)[face_orders]) | |
| chosen_rectangle_pos_list = [chosen_rectangle_pos for _, chosen_rectangle_pos in sorted(zip(face_orders, chosen_rectangle_pos_list))] | |
| return face_detection_infos, face_direction_infos, face_position_infos, face_coordinate_infos, face_pose_infos, face_area_infos, chosen_rectangle_pos_list | |
| def detect_landmarks_from_image(image, face_mesh, img_size, head_pose_info, get_one_landmark=False): | |
| """ | |
| Input: | |
| `image` (np.ndarray): Image for face landmarks detection | |
| `img_size` (int, int): shape of the image (width, height) | |
| `face_mesh` (mp.solutions.face_mesh): landmarks detection model | |
| `get_one_landmark` (bool): In case you just want predict one landmarks for one face, `True` for return one landmarks infos, `False` otherwise | |
| `head_pose_info` (dict): head pose configs dictionary | |
| Return: | |
| `face_pose_infos` (list(list[float])): pose info for each face (x,y,z) | |
| `face_direction_infos` (list[str]): direction for each face | |
| """ | |
| image_resize = resize_img(image, img_size) | |
| results = face_mesh.process(image_resize) | |
| # Multi landmarks informations from landmarks detection model | |
| multi_face_landmarks = results.multi_face_landmarks | |
| if not multi_face_landmarks: | |
| return [(0, 0, 0)], ['NO FACE'] | |
| else: | |
| face_pose_infos, face_direction_infos = [], [] | |
| img_w, img_h = img_size | |
| face_3d = [] | |
| face_2d = [] | |
| for face_landmarks in multi_face_landmarks: | |
| for idx, lm in enumerate(face_landmarks.landmark): | |
| if idx == 33 or idx == 263 or idx == 1 or idx == 61 or idx == 291 or idx == 199: | |
| if idx == 1: | |
| nose_2d = (lm.x * img_w, lm.y * img_h) | |
| nose_3d = (lm.x * img_w, lm.y * img_h, lm.z * 3000) | |
| x, y = int(lm.x * img_w), int(lm.y * img_h) | |
| # Get the 2D Coordinates | |
| face_2d.append([x, y]) | |
| # Get the 3D Coordinates | |
| face_3d.append([x, y, lm.z]) | |
| # Convert to Numpy array | |
| face_2d = np.array(face_2d, dtype=np.float64) | |
| face_3d = np.array(face_3d, dtype=np.float64) | |
| # The camera matrix | |
| focal_length = 1 * img_w | |
| cam_matrix = np.array([ [focal_length, 0, img_h/2], | |
| [0, focal_length, img_w/2], | |
| [0, 0, 1]]) | |
| # The distortion parameters | |
| dist_matrix = np.zeros((4, 1), dtype=np.float64) | |
| # Solve PnP | |
| success, rot_vec, trans_vec = cv2.solvePnP(face_3d, face_2d, cam_matrix,dist_matrix) | |
| # Get rotational matrix | |
| rmat, jac = cv2.Rodrigues(rot_vec) | |
| # Get angles | |
| angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat) | |
| # Get the y rotation degree | |
| x = angles[0] * 360 | |
| y = angles[1] * 360 | |
| z = angles[2] * 360 | |
| face_pose_infos.append((x,y,z)) | |
| # See where the user's head tilting | |
| face_direction = 'NOT FRONT' | |
| for head_pose_direction_name, head_pose_direction in head_pose_info['DIRECTION'].items(): | |
| if (head_pose_direction[0][0] < x and x <= head_pose_direction[1][0]) and \ | |
| (head_pose_direction[0][1] < y and y <= head_pose_direction[1][1]): | |
| face_direction = head_pose_direction_name | |
| break | |
| face_direction_infos.append(face_direction) | |
| # Display the nose direction | |
| nose_3d_projection, jacobian = cv2.projectPoints(nose_3d, rot_vec, trans_vec, cam_matrix, dist_matrix) | |
| p1 = (int(nose_2d[0]), int(nose_2d[1])) | |
| p2 = (int(nose_2d[0] + y*10), int(nose_2d[1] - x*10)) | |
| # cv2.line(image_face, p1, p2, (255, 0, 0), 3) | |
| if get_one_landmark: | |
| break | |
| return face_pose_infos, face_direction_infos |