Spaces:
Runtime error
Runtime error
Modify 'show face keypoints' and 'visualize coordinate' now can be adjusted
Browse files- app.py +11 -6
- drawing_utils.py +297 -0
app.py
CHANGED
|
@@ -9,6 +9,7 @@ import yaml
|
|
| 9 |
from PIL import Image
|
| 10 |
|
| 11 |
import streamlit as st
|
|
|
|
| 12 |
st.set_page_config(layout="wide")
|
| 13 |
|
| 14 |
|
|
@@ -54,7 +55,9 @@ with left_col:
|
|
| 54 |
# if st.session_state.img_path is not None:
|
| 55 |
# st.write("Your image data source: **" + st.session_state.img_path + "**")
|
| 56 |
|
| 57 |
-
st.checkbox("Using face meshgrid for visualization", key="use_face_mesh")
|
|
|
|
|
|
|
| 58 |
st.radio(
|
| 59 |
"Select sample size for model input and save output:",
|
| 60 |
key="img_size",
|
|
@@ -87,9 +90,10 @@ if st.session_state.img_upload is not None:
|
|
| 87 |
mp_face_detection = mp.solutions.face_detection
|
| 88 |
face_mesh = mp_face_mesh.FaceMesh(min_detection_confidence=.5, min_tracking_confidence=0.5)
|
| 89 |
|
| 90 |
-
mp_drawing = mp.solutions.drawing_utils
|
| 91 |
|
| 92 |
-
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
|
|
|
|
| 93 |
|
| 94 |
with open(args.head_pose_info, 'r') as f:
|
| 95 |
head_pose_info = yaml.load(f, Loader=yaml.FullLoader)
|
|
@@ -170,7 +174,7 @@ if st.session_state.img_upload is not None:
|
|
| 170 |
for face_landmarks in results.multi_face_landmarks:
|
| 171 |
# Draw face mesh grid
|
| 172 |
if st.session_state.use_face_mesh:
|
| 173 |
-
|
| 174 |
image=image_face,
|
| 175 |
landmark_list=face_landmarks,
|
| 176 |
connections=mp_face_mesh.FACEMESH_CONTOURS,
|
|
@@ -245,7 +249,8 @@ if st.session_state.img_upload is not None:
|
|
| 245 |
|
| 246 |
# Add the text on the image
|
| 247 |
cv2.putText(image, text, (xmin, ymin), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 3)
|
| 248 |
-
|
|
|
|
| 249 |
|
| 250 |
end = time.time()
|
| 251 |
cur_time = end - start
|
|
@@ -259,7 +264,7 @@ if st.session_state.img_upload is not None:
|
|
| 259 |
|
| 260 |
if detection_results.detections:
|
| 261 |
for face_id, detection in enumerate(detection_results.detections):
|
| 262 |
-
|
| 263 |
|
| 264 |
with right_col:
|
| 265 |
st.image(image_results, width=st.session_state.width_visual, caption=image_captions)
|
|
|
|
| 9 |
from PIL import Image
|
| 10 |
|
| 11 |
import streamlit as st
|
| 12 |
+
from drawing_utils import draw_landmarks, draw_detection, DrawingSpec
|
| 13 |
st.set_page_config(layout="wide")
|
| 14 |
|
| 15 |
|
|
|
|
| 55 |
# if st.session_state.img_path is not None:
|
| 56 |
# st.write("Your image data source: **" + st.session_state.img_path + "**")
|
| 57 |
|
| 58 |
+
st.checkbox("Using **face meshgrid** for visualization", key="use_face_mesh")
|
| 59 |
+
st.checkbox("Using **face keypoints** for visualization", key="use_face_keypoints")
|
| 60 |
+
st.checkbox("Viualize **coordinate** of face keypoints", key="show_coordinate")
|
| 61 |
st.radio(
|
| 62 |
"Select sample size for model input and save output:",
|
| 63 |
key="img_size",
|
|
|
|
| 90 |
mp_face_detection = mp.solutions.face_detection
|
| 91 |
face_mesh = mp_face_mesh.FaceMesh(min_detection_confidence=.5, min_tracking_confidence=0.5)
|
| 92 |
|
| 93 |
+
# mp_drawing = mp.solutions.drawing_utils
|
| 94 |
|
| 95 |
+
# drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
|
| 96 |
+
drawing_spec = DrawingSpec(thickness=1, circle_radius=1)
|
| 97 |
|
| 98 |
with open(args.head_pose_info, 'r') as f:
|
| 99 |
head_pose_info = yaml.load(f, Loader=yaml.FullLoader)
|
|
|
|
| 174 |
for face_landmarks in results.multi_face_landmarks:
|
| 175 |
# Draw face mesh grid
|
| 176 |
if st.session_state.use_face_mesh:
|
| 177 |
+
draw_landmarks(
|
| 178 |
image=image_face,
|
| 179 |
landmark_list=face_landmarks,
|
| 180 |
connections=mp_face_mesh.FACEMESH_CONTOURS,
|
|
|
|
| 249 |
|
| 250 |
# Add the text on the image
|
| 251 |
cv2.putText(image, text, (xmin, ymin), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 3)
|
| 252 |
+
if st.session_state.show_coordinate:
|
| 253 |
+
cv2.putText(image, f"({str(np.round(x, 2))},{str(np.round(y, 2))},{str(np.round(z, 2))})", (xmax-150, ymax+20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
|
| 254 |
|
| 255 |
end = time.time()
|
| 256 |
cur_time = end - start
|
|
|
|
| 264 |
|
| 265 |
if detection_results.detections:
|
| 266 |
for face_id, detection in enumerate(detection_results.detections):
|
| 267 |
+
draw_detection(image, detection, draw_keypoints=st.session_state.use_face_keypoints)
|
| 268 |
|
| 269 |
with right_col:
|
| 270 |
st.image(image_results, width=st.session_state.width_visual, caption=image_captions)
|
drawing_utils.py
ADDED
|
@@ -0,0 +1,297 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MediaPipe solution drawing utils."""
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
from typing import List, Mapping, Optional, Tuple, Union
|
| 5 |
+
|
| 6 |
+
import cv2
|
| 7 |
+
import dataclasses
|
| 8 |
+
import matplotlib.pyplot as plt
|
| 9 |
+
import numpy as np
|
| 10 |
+
|
| 11 |
+
from mediapipe.framework.formats import detection_pb2
|
| 12 |
+
from mediapipe.framework.formats import location_data_pb2
|
| 13 |
+
from mediapipe.framework.formats import landmark_pb2
|
| 14 |
+
|
| 15 |
+
_PRESENCE_THRESHOLD = 0.5
|
| 16 |
+
_VISIBILITY_THRESHOLD = 0.5
|
| 17 |
+
_BGR_CHANNELS = 3
|
| 18 |
+
|
| 19 |
+
WHITE_COLOR = (224, 224, 224)
|
| 20 |
+
BLACK_COLOR = (0, 0, 0)
|
| 21 |
+
RED_COLOR = (0, 0, 255)
|
| 22 |
+
GREEN_COLOR = (0, 128, 0)
|
| 23 |
+
BLUE_COLOR = (255, 0, 0)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@dataclasses.dataclass
|
| 27 |
+
class DrawingSpec:
|
| 28 |
+
# Color for drawing the annotation. Default to the white color.
|
| 29 |
+
color: Tuple[int, int, int] = WHITE_COLOR
|
| 30 |
+
# Thickness for drawing the annotation. Default to 2 pixels.
|
| 31 |
+
thickness: int = 2
|
| 32 |
+
# Circle radius. Default to 2 pixels.
|
| 33 |
+
circle_radius: int = 2
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _normalized_to_pixel_coordinates(
|
| 37 |
+
normalized_x: float, normalized_y: float, image_width: int,
|
| 38 |
+
image_height: int) -> Union[None, Tuple[int, int]]:
|
| 39 |
+
"""Converts normalized value pair to pixel coordinates."""
|
| 40 |
+
|
| 41 |
+
# Checks if the float value is between 0 and 1.
|
| 42 |
+
def is_valid_normalized_value(value: float) -> bool:
|
| 43 |
+
return (value > 0 or math.isclose(0, value)) and (value < 1 or
|
| 44 |
+
math.isclose(1, value))
|
| 45 |
+
|
| 46 |
+
if not (is_valid_normalized_value(normalized_x) and
|
| 47 |
+
is_valid_normalized_value(normalized_y)):
|
| 48 |
+
# TODO: Draw coordinates even if it's outside of the image bounds.
|
| 49 |
+
return None
|
| 50 |
+
x_px = min(math.floor(normalized_x * image_width), image_width - 1)
|
| 51 |
+
y_px = min(math.floor(normalized_y * image_height), image_height - 1)
|
| 52 |
+
return x_px, y_px
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def draw_detection(
|
| 56 |
+
image: np.ndarray,
|
| 57 |
+
detection: detection_pb2.Detection,
|
| 58 |
+
keypoint_drawing_spec: DrawingSpec = DrawingSpec(color=RED_COLOR),
|
| 59 |
+
bbox_drawing_spec: DrawingSpec = DrawingSpec(),
|
| 60 |
+
draw_keypoints: bool = True):
|
| 61 |
+
"""Draws the detction bounding box and keypoints on the image.
|
| 62 |
+
Args:
|
| 63 |
+
image: A three channel BGR image represented as numpy ndarray.
|
| 64 |
+
detection: A detection proto message to be annotated on the image.
|
| 65 |
+
keypoint_drawing_spec: A DrawingSpec object that specifies the keypoints'
|
| 66 |
+
drawing settings such as color, line thickness, and circle radius.
|
| 67 |
+
bbox_drawing_spec: A DrawingSpec object that specifies the bounding box's
|
| 68 |
+
drawing settings such as color and line thickness.
|
| 69 |
+
Raises:
|
| 70 |
+
ValueError: If one of the followings:
|
| 71 |
+
a) If the input image is not three channel BGR.
|
| 72 |
+
b) If the location data is not relative data.
|
| 73 |
+
"""
|
| 74 |
+
if not detection.location_data:
|
| 75 |
+
return
|
| 76 |
+
if image.shape[2] != _BGR_CHANNELS:
|
| 77 |
+
raise ValueError('Input image must contain three channel bgr data.')
|
| 78 |
+
image_rows, image_cols, _ = image.shape
|
| 79 |
+
|
| 80 |
+
location = detection.location_data
|
| 81 |
+
if location.format != location_data_pb2.LocationData.RELATIVE_BOUNDING_BOX:
|
| 82 |
+
raise ValueError(
|
| 83 |
+
'LocationData must be relative for this drawing funtion to work.')
|
| 84 |
+
# Draws keypoints.
|
| 85 |
+
if draw_keypoints:
|
| 86 |
+
for keypoint in location.relative_keypoints:
|
| 87 |
+
keypoint_px = _normalized_to_pixel_coordinates(keypoint.x, keypoint.y,
|
| 88 |
+
image_cols, image_rows)
|
| 89 |
+
cv2.circle(image, keypoint_px, keypoint_drawing_spec.circle_radius,
|
| 90 |
+
keypoint_drawing_spec.color, keypoint_drawing_spec.thickness)
|
| 91 |
+
# Draws bounding box if exists.
|
| 92 |
+
if not location.HasField('relative_bounding_box'):
|
| 93 |
+
return
|
| 94 |
+
relative_bounding_box = location.relative_bounding_box
|
| 95 |
+
rect_start_point = _normalized_to_pixel_coordinates(
|
| 96 |
+
relative_bounding_box.xmin, relative_bounding_box.ymin, image_cols,
|
| 97 |
+
image_rows)
|
| 98 |
+
rect_end_point = _normalized_to_pixel_coordinates(
|
| 99 |
+
relative_bounding_box.xmin + relative_bounding_box.width,
|
| 100 |
+
relative_bounding_box.ymin + relative_bounding_box.height, image_cols,
|
| 101 |
+
image_rows)
|
| 102 |
+
cv2.rectangle(image, rect_start_point, rect_end_point,
|
| 103 |
+
bbox_drawing_spec.color, bbox_drawing_spec.thickness)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def draw_landmarks(
|
| 107 |
+
image: np.ndarray,
|
| 108 |
+
landmark_list: landmark_pb2.NormalizedLandmarkList,
|
| 109 |
+
connections: Optional[List[Tuple[int, int]]] = None,
|
| 110 |
+
landmark_drawing_spec: Union[DrawingSpec,
|
| 111 |
+
Mapping[int, DrawingSpec]] = DrawingSpec(
|
| 112 |
+
color=RED_COLOR),
|
| 113 |
+
connection_drawing_spec: Union[DrawingSpec,
|
| 114 |
+
Mapping[Tuple[int, int],
|
| 115 |
+
DrawingSpec]] = DrawingSpec()):
|
| 116 |
+
"""Draws the landmarks and the connections on the image.
|
| 117 |
+
Args:
|
| 118 |
+
image: A three channel BGR image represented as numpy ndarray.
|
| 119 |
+
landmark_list: A normalized landmark list proto message to be annotated on
|
| 120 |
+
the image.
|
| 121 |
+
connections: A list of landmark index tuples that specifies how landmarks to
|
| 122 |
+
be connected in the drawing.
|
| 123 |
+
landmark_drawing_spec: Either a DrawingSpec object or a mapping from hand
|
| 124 |
+
landmarks to the DrawingSpecs that specifies the landmarks' drawing
|
| 125 |
+
settings such as color, line thickness, and circle radius. If this
|
| 126 |
+
argument is explicitly set to None, no landmarks will be drawn.
|
| 127 |
+
connection_drawing_spec: Either a DrawingSpec object or a mapping from hand
|
| 128 |
+
connections to the DrawingSpecs that specifies the connections' drawing
|
| 129 |
+
settings such as color and line thickness. If this argument is explicitly
|
| 130 |
+
set to None, no landmark connections will be drawn.
|
| 131 |
+
Raises:
|
| 132 |
+
ValueError: If one of the followings:
|
| 133 |
+
a) If the input image is not three channel BGR.
|
| 134 |
+
b) If any connetions contain invalid landmark index.
|
| 135 |
+
"""
|
| 136 |
+
if not landmark_list:
|
| 137 |
+
return
|
| 138 |
+
if image.shape[2] != _BGR_CHANNELS:
|
| 139 |
+
raise ValueError('Input image must contain three channel bgr data.')
|
| 140 |
+
image_rows, image_cols, _ = image.shape
|
| 141 |
+
idx_to_coordinates = {}
|
| 142 |
+
for idx, landmark in enumerate(landmark_list.landmark):
|
| 143 |
+
if ((landmark.HasField('visibility') and
|
| 144 |
+
landmark.visibility < _VISIBILITY_THRESHOLD) or
|
| 145 |
+
(landmark.HasField('presence') and
|
| 146 |
+
landmark.presence < _PRESENCE_THRESHOLD)):
|
| 147 |
+
continue
|
| 148 |
+
landmark_px = _normalized_to_pixel_coordinates(landmark.x, landmark.y,
|
| 149 |
+
image_cols, image_rows)
|
| 150 |
+
if landmark_px:
|
| 151 |
+
idx_to_coordinates[idx] = landmark_px
|
| 152 |
+
if connections:
|
| 153 |
+
num_landmarks = len(landmark_list.landmark)
|
| 154 |
+
# Draws the connections if the start and end landmarks are both visible.
|
| 155 |
+
for connection in connections:
|
| 156 |
+
start_idx = connection[0]
|
| 157 |
+
end_idx = connection[1]
|
| 158 |
+
if not (0 <= start_idx < num_landmarks and 0 <= end_idx < num_landmarks):
|
| 159 |
+
raise ValueError(f'Landmark index is out of range. Invalid connection '
|
| 160 |
+
f'from landmark #{start_idx} to landmark #{end_idx}.')
|
| 161 |
+
if start_idx in idx_to_coordinates and end_idx in idx_to_coordinates:
|
| 162 |
+
drawing_spec = connection_drawing_spec[connection] if isinstance(
|
| 163 |
+
connection_drawing_spec, Mapping) else connection_drawing_spec
|
| 164 |
+
cv2.line(image, idx_to_coordinates[start_idx],
|
| 165 |
+
idx_to_coordinates[end_idx], drawing_spec.color,
|
| 166 |
+
drawing_spec.thickness)
|
| 167 |
+
# Draws landmark points after finishing the connection lines, which is
|
| 168 |
+
# aesthetically better.
|
| 169 |
+
if landmark_drawing_spec:
|
| 170 |
+
for idx, landmark_px in idx_to_coordinates.items():
|
| 171 |
+
drawing_spec = landmark_drawing_spec[idx] if isinstance(
|
| 172 |
+
landmark_drawing_spec, Mapping) else landmark_drawing_spec
|
| 173 |
+
# White circle border
|
| 174 |
+
circle_border_radius = max(drawing_spec.circle_radius + 1,
|
| 175 |
+
int(drawing_spec.circle_radius * 1.2))
|
| 176 |
+
cv2.circle(image, landmark_px, circle_border_radius, WHITE_COLOR,
|
| 177 |
+
drawing_spec.thickness)
|
| 178 |
+
# Fill color into the circle
|
| 179 |
+
cv2.circle(image, landmark_px, drawing_spec.circle_radius,
|
| 180 |
+
drawing_spec.color, drawing_spec.thickness)
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def draw_axis(image: np.ndarray,
|
| 184 |
+
rotation: np.ndarray,
|
| 185 |
+
translation: np.ndarray,
|
| 186 |
+
focal_length: Tuple[float, float] = (1.0, 1.0),
|
| 187 |
+
principal_point: Tuple[float, float] = (0.0, 0.0),
|
| 188 |
+
axis_length: float = 0.1,
|
| 189 |
+
axis_drawing_spec: DrawingSpec = DrawingSpec()):
|
| 190 |
+
"""Draws the 3D axis on the image.
|
| 191 |
+
Args:
|
| 192 |
+
image: A three channel BGR image represented as numpy ndarray.
|
| 193 |
+
rotation: Rotation matrix from object to camera coordinate frame.
|
| 194 |
+
translation: Translation vector from object to camera coordinate frame.
|
| 195 |
+
focal_length: camera focal length along x and y directions.
|
| 196 |
+
principal_point: camera principal point in x and y.
|
| 197 |
+
axis_length: length of the axis in the drawing.
|
| 198 |
+
axis_drawing_spec: A DrawingSpec object that specifies the xyz axis drawing
|
| 199 |
+
settings such as line thickness.
|
| 200 |
+
Raises:
|
| 201 |
+
ValueError: If one of the followings:
|
| 202 |
+
a) If the input image is not three channel BGR.
|
| 203 |
+
"""
|
| 204 |
+
if image.shape[2] != _BGR_CHANNELS:
|
| 205 |
+
raise ValueError('Input image must contain three channel bgr data.')
|
| 206 |
+
image_rows, image_cols, _ = image.shape
|
| 207 |
+
# Create axis points in camera coordinate frame.
|
| 208 |
+
axis_world = np.float32([[0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1]])
|
| 209 |
+
axis_cam = np.matmul(rotation, axis_length * axis_world.T).T + translation
|
| 210 |
+
x = axis_cam[..., 0]
|
| 211 |
+
y = axis_cam[..., 1]
|
| 212 |
+
z = axis_cam[..., 2]
|
| 213 |
+
# Project 3D points to NDC space.
|
| 214 |
+
fx, fy = focal_length
|
| 215 |
+
px, py = principal_point
|
| 216 |
+
x_ndc = np.clip(-fx * x / (z + 1e-5) + px, -1., 1.)
|
| 217 |
+
y_ndc = np.clip(-fy * y / (z + 1e-5) + py, -1., 1.)
|
| 218 |
+
# Convert from NDC space to image space.
|
| 219 |
+
x_im = np.int32((1 + x_ndc) * 0.5 * image_cols)
|
| 220 |
+
y_im = np.int32((1 - y_ndc) * 0.5 * image_rows)
|
| 221 |
+
# Draw xyz axis on the image.
|
| 222 |
+
origin = (x_im[0], y_im[0])
|
| 223 |
+
x_axis = (x_im[1], y_im[1])
|
| 224 |
+
y_axis = (x_im[2], y_im[2])
|
| 225 |
+
z_axis = (x_im[3], y_im[3])
|
| 226 |
+
cv2.arrowedLine(image, origin, x_axis, RED_COLOR, axis_drawing_spec.thickness)
|
| 227 |
+
cv2.arrowedLine(image, origin, y_axis, GREEN_COLOR,
|
| 228 |
+
axis_drawing_spec.thickness)
|
| 229 |
+
cv2.arrowedLine(image, origin, z_axis, BLUE_COLOR,
|
| 230 |
+
axis_drawing_spec.thickness)
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def _normalize_color(color):
|
| 234 |
+
return tuple(v / 255. for v in color)
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def plot_landmarks(landmark_list: landmark_pb2.NormalizedLandmarkList,
|
| 238 |
+
connections: Optional[List[Tuple[int, int]]] = None,
|
| 239 |
+
landmark_drawing_spec: DrawingSpec = DrawingSpec(
|
| 240 |
+
color=RED_COLOR, thickness=5),
|
| 241 |
+
connection_drawing_spec: DrawingSpec = DrawingSpec(
|
| 242 |
+
color=BLACK_COLOR, thickness=5),
|
| 243 |
+
elevation: int = 10,
|
| 244 |
+
azimuth: int = 10):
|
| 245 |
+
"""Plot the landmarks and the connections in matplotlib 3d.
|
| 246 |
+
Args:
|
| 247 |
+
landmark_list: A normalized landmark list proto message to be plotted.
|
| 248 |
+
connections: A list of landmark index tuples that specifies how landmarks to
|
| 249 |
+
be connected.
|
| 250 |
+
landmark_drawing_spec: A DrawingSpec object that specifies the landmarks'
|
| 251 |
+
drawing settings such as color and line thickness.
|
| 252 |
+
connection_drawing_spec: A DrawingSpec object that specifies the
|
| 253 |
+
connections' drawing settings such as color and line thickness.
|
| 254 |
+
elevation: The elevation from which to view the plot.
|
| 255 |
+
azimuth: the azimuth angle to rotate the plot.
|
| 256 |
+
Raises:
|
| 257 |
+
ValueError: If any connection contains an invalid landmark index.
|
| 258 |
+
"""
|
| 259 |
+
if not landmark_list:
|
| 260 |
+
return
|
| 261 |
+
plt.figure(figsize=(10, 10))
|
| 262 |
+
ax = plt.axes(projection='3d')
|
| 263 |
+
ax.view_init(elev=elevation, azim=azimuth)
|
| 264 |
+
plotted_landmarks = {}
|
| 265 |
+
for idx, landmark in enumerate(landmark_list.landmark):
|
| 266 |
+
if ((landmark.HasField('visibility') and
|
| 267 |
+
landmark.visibility < _VISIBILITY_THRESHOLD) or
|
| 268 |
+
(landmark.HasField('presence') and
|
| 269 |
+
landmark.presence < _PRESENCE_THRESHOLD)):
|
| 270 |
+
continue
|
| 271 |
+
ax.scatter3D(
|
| 272 |
+
xs=[-landmark.z],
|
| 273 |
+
ys=[landmark.x],
|
| 274 |
+
zs=[-landmark.y],
|
| 275 |
+
color=_normalize_color(landmark_drawing_spec.color[::-1]),
|
| 276 |
+
linewidth=landmark_drawing_spec.thickness)
|
| 277 |
+
plotted_landmarks[idx] = (-landmark.z, landmark.x, -landmark.y)
|
| 278 |
+
if connections:
|
| 279 |
+
num_landmarks = len(landmark_list.landmark)
|
| 280 |
+
# Draws the connections if the start and end landmarks are both visible.
|
| 281 |
+
for connection in connections:
|
| 282 |
+
start_idx = connection[0]
|
| 283 |
+
end_idx = connection[1]
|
| 284 |
+
if not (0 <= start_idx < num_landmarks and 0 <= end_idx < num_landmarks):
|
| 285 |
+
raise ValueError(f'Landmark index is out of range. Invalid connection '
|
| 286 |
+
f'from landmark #{start_idx} to landmark #{end_idx}.')
|
| 287 |
+
if start_idx in plotted_landmarks and end_idx in plotted_landmarks:
|
| 288 |
+
landmark_pair = [
|
| 289 |
+
plotted_landmarks[start_idx], plotted_landmarks[end_idx]
|
| 290 |
+
]
|
| 291 |
+
ax.plot3D(
|
| 292 |
+
xs=[landmark_pair[0][0], landmark_pair[1][0]],
|
| 293 |
+
ys=[landmark_pair[0][1], landmark_pair[1][1]],
|
| 294 |
+
zs=[landmark_pair[0][2], landmark_pair[1][2]],
|
| 295 |
+
color=_normalize_color(connection_drawing_spec.color[::-1]),
|
| 296 |
+
linewidth=connection_drawing_spec.thickness)
|
| 297 |
+
plt.show()
|