manhdo commited on
Commit
c207cc1
·
1 Parent(s): bf0efa7

Modify 'show face keypoints' and 'visualize coordinate' now can be adjusted

Browse files
Files changed (2) hide show
  1. app.py +11 -6
  2. drawing_utils.py +297 -0
app.py CHANGED
@@ -9,6 +9,7 @@ import yaml
9
  from PIL import Image
10
 
11
  import streamlit as st
 
12
  st.set_page_config(layout="wide")
13
 
14
 
@@ -54,7 +55,9 @@ with left_col:
54
  # if st.session_state.img_path is not None:
55
  # st.write("Your image data source: **" + st.session_state.img_path + "**")
56
 
57
- st.checkbox("Using face meshgrid for visualization", key="use_face_mesh")
 
 
58
  st.radio(
59
  "Select sample size for model input and save output:",
60
  key="img_size",
@@ -87,9 +90,10 @@ if st.session_state.img_upload is not None:
87
  mp_face_detection = mp.solutions.face_detection
88
  face_mesh = mp_face_mesh.FaceMesh(min_detection_confidence=.5, min_tracking_confidence=0.5)
89
 
90
- mp_drawing = mp.solutions.drawing_utils
91
 
92
- drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
 
93
 
94
  with open(args.head_pose_info, 'r') as f:
95
  head_pose_info = yaml.load(f, Loader=yaml.FullLoader)
@@ -170,7 +174,7 @@ if st.session_state.img_upload is not None:
170
  for face_landmarks in results.multi_face_landmarks:
171
  # Draw face mesh grid
172
  if st.session_state.use_face_mesh:
173
- mp_drawing.draw_landmarks(
174
  image=image_face,
175
  landmark_list=face_landmarks,
176
  connections=mp_face_mesh.FACEMESH_CONTOURS,
@@ -245,7 +249,8 @@ if st.session_state.img_upload is not None:
245
 
246
  # Add the text on the image
247
  cv2.putText(image, text, (xmin, ymin), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 3)
248
- cv2.putText(image, f"({str(np.round(x, 2))},{str(np.round(y, 2))},{str(np.round(z, 2))})", (xmax-150, ymax+20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
 
249
 
250
  end = time.time()
251
  cur_time = end - start
@@ -259,7 +264,7 @@ if st.session_state.img_upload is not None:
259
 
260
  if detection_results.detections:
261
  for face_id, detection in enumerate(detection_results.detections):
262
- mp_drawing.draw_detection(image, detection)
263
 
264
  with right_col:
265
  st.image(image_results, width=st.session_state.width_visual, caption=image_captions)
 
9
  from PIL import Image
10
 
11
  import streamlit as st
12
+ from drawing_utils import draw_landmarks, draw_detection, DrawingSpec
13
  st.set_page_config(layout="wide")
14
 
15
 
 
55
  # if st.session_state.img_path is not None:
56
  # st.write("Your image data source: **" + st.session_state.img_path + "**")
57
 
58
+ st.checkbox("Using **face meshgrid** for visualization", key="use_face_mesh")
59
+ st.checkbox("Using **face keypoints** for visualization", key="use_face_keypoints")
60
+ st.checkbox("Viualize **coordinate** of face keypoints", key="show_coordinate")
61
  st.radio(
62
  "Select sample size for model input and save output:",
63
  key="img_size",
 
90
  mp_face_detection = mp.solutions.face_detection
91
  face_mesh = mp_face_mesh.FaceMesh(min_detection_confidence=.5, min_tracking_confidence=0.5)
92
 
93
+ # mp_drawing = mp.solutions.drawing_utils
94
 
95
+ # drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
96
+ drawing_spec = DrawingSpec(thickness=1, circle_radius=1)
97
 
98
  with open(args.head_pose_info, 'r') as f:
99
  head_pose_info = yaml.load(f, Loader=yaml.FullLoader)
 
174
  for face_landmarks in results.multi_face_landmarks:
175
  # Draw face mesh grid
176
  if st.session_state.use_face_mesh:
177
+ draw_landmarks(
178
  image=image_face,
179
  landmark_list=face_landmarks,
180
  connections=mp_face_mesh.FACEMESH_CONTOURS,
 
249
 
250
  # Add the text on the image
251
  cv2.putText(image, text, (xmin, ymin), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 3)
252
+ if st.session_state.show_coordinate:
253
+ cv2.putText(image, f"({str(np.round(x, 2))},{str(np.round(y, 2))},{str(np.round(z, 2))})", (xmax-150, ymax+20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
254
 
255
  end = time.time()
256
  cur_time = end - start
 
264
 
265
  if detection_results.detections:
266
  for face_id, detection in enumerate(detection_results.detections):
267
+ draw_detection(image, detection, draw_keypoints=st.session_state.use_face_keypoints)
268
 
269
  with right_col:
270
  st.image(image_results, width=st.session_state.width_visual, caption=image_captions)
drawing_utils.py ADDED
@@ -0,0 +1,297 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """MediaPipe solution drawing utils."""
2
+
3
+ import math
4
+ from typing import List, Mapping, Optional, Tuple, Union
5
+
6
+ import cv2
7
+ import dataclasses
8
+ import matplotlib.pyplot as plt
9
+ import numpy as np
10
+
11
+ from mediapipe.framework.formats import detection_pb2
12
+ from mediapipe.framework.formats import location_data_pb2
13
+ from mediapipe.framework.formats import landmark_pb2
14
+
15
+ _PRESENCE_THRESHOLD = 0.5
16
+ _VISIBILITY_THRESHOLD = 0.5
17
+ _BGR_CHANNELS = 3
18
+
19
+ WHITE_COLOR = (224, 224, 224)
20
+ BLACK_COLOR = (0, 0, 0)
21
+ RED_COLOR = (0, 0, 255)
22
+ GREEN_COLOR = (0, 128, 0)
23
+ BLUE_COLOR = (255, 0, 0)
24
+
25
+
26
+ @dataclasses.dataclass
27
+ class DrawingSpec:
28
+ # Color for drawing the annotation. Default to the white color.
29
+ color: Tuple[int, int, int] = WHITE_COLOR
30
+ # Thickness for drawing the annotation. Default to 2 pixels.
31
+ thickness: int = 2
32
+ # Circle radius. Default to 2 pixels.
33
+ circle_radius: int = 2
34
+
35
+
36
+ def _normalized_to_pixel_coordinates(
37
+ normalized_x: float, normalized_y: float, image_width: int,
38
+ image_height: int) -> Union[None, Tuple[int, int]]:
39
+ """Converts normalized value pair to pixel coordinates."""
40
+
41
+ # Checks if the float value is between 0 and 1.
42
+ def is_valid_normalized_value(value: float) -> bool:
43
+ return (value > 0 or math.isclose(0, value)) and (value < 1 or
44
+ math.isclose(1, value))
45
+
46
+ if not (is_valid_normalized_value(normalized_x) and
47
+ is_valid_normalized_value(normalized_y)):
48
+ # TODO: Draw coordinates even if it's outside of the image bounds.
49
+ return None
50
+ x_px = min(math.floor(normalized_x * image_width), image_width - 1)
51
+ y_px = min(math.floor(normalized_y * image_height), image_height - 1)
52
+ return x_px, y_px
53
+
54
+
55
+ def draw_detection(
56
+ image: np.ndarray,
57
+ detection: detection_pb2.Detection,
58
+ keypoint_drawing_spec: DrawingSpec = DrawingSpec(color=RED_COLOR),
59
+ bbox_drawing_spec: DrawingSpec = DrawingSpec(),
60
+ draw_keypoints: bool = True):
61
+ """Draws the detction bounding box and keypoints on the image.
62
+ Args:
63
+ image: A three channel BGR image represented as numpy ndarray.
64
+ detection: A detection proto message to be annotated on the image.
65
+ keypoint_drawing_spec: A DrawingSpec object that specifies the keypoints'
66
+ drawing settings such as color, line thickness, and circle radius.
67
+ bbox_drawing_spec: A DrawingSpec object that specifies the bounding box's
68
+ drawing settings such as color and line thickness.
69
+ Raises:
70
+ ValueError: If one of the followings:
71
+ a) If the input image is not three channel BGR.
72
+ b) If the location data is not relative data.
73
+ """
74
+ if not detection.location_data:
75
+ return
76
+ if image.shape[2] != _BGR_CHANNELS:
77
+ raise ValueError('Input image must contain three channel bgr data.')
78
+ image_rows, image_cols, _ = image.shape
79
+
80
+ location = detection.location_data
81
+ if location.format != location_data_pb2.LocationData.RELATIVE_BOUNDING_BOX:
82
+ raise ValueError(
83
+ 'LocationData must be relative for this drawing funtion to work.')
84
+ # Draws keypoints.
85
+ if draw_keypoints:
86
+ for keypoint in location.relative_keypoints:
87
+ keypoint_px = _normalized_to_pixel_coordinates(keypoint.x, keypoint.y,
88
+ image_cols, image_rows)
89
+ cv2.circle(image, keypoint_px, keypoint_drawing_spec.circle_radius,
90
+ keypoint_drawing_spec.color, keypoint_drawing_spec.thickness)
91
+ # Draws bounding box if exists.
92
+ if not location.HasField('relative_bounding_box'):
93
+ return
94
+ relative_bounding_box = location.relative_bounding_box
95
+ rect_start_point = _normalized_to_pixel_coordinates(
96
+ relative_bounding_box.xmin, relative_bounding_box.ymin, image_cols,
97
+ image_rows)
98
+ rect_end_point = _normalized_to_pixel_coordinates(
99
+ relative_bounding_box.xmin + relative_bounding_box.width,
100
+ relative_bounding_box.ymin + relative_bounding_box.height, image_cols,
101
+ image_rows)
102
+ cv2.rectangle(image, rect_start_point, rect_end_point,
103
+ bbox_drawing_spec.color, bbox_drawing_spec.thickness)
104
+
105
+
106
+ def draw_landmarks(
107
+ image: np.ndarray,
108
+ landmark_list: landmark_pb2.NormalizedLandmarkList,
109
+ connections: Optional[List[Tuple[int, int]]] = None,
110
+ landmark_drawing_spec: Union[DrawingSpec,
111
+ Mapping[int, DrawingSpec]] = DrawingSpec(
112
+ color=RED_COLOR),
113
+ connection_drawing_spec: Union[DrawingSpec,
114
+ Mapping[Tuple[int, int],
115
+ DrawingSpec]] = DrawingSpec()):
116
+ """Draws the landmarks and the connections on the image.
117
+ Args:
118
+ image: A three channel BGR image represented as numpy ndarray.
119
+ landmark_list: A normalized landmark list proto message to be annotated on
120
+ the image.
121
+ connections: A list of landmark index tuples that specifies how landmarks to
122
+ be connected in the drawing.
123
+ landmark_drawing_spec: Either a DrawingSpec object or a mapping from hand
124
+ landmarks to the DrawingSpecs that specifies the landmarks' drawing
125
+ settings such as color, line thickness, and circle radius. If this
126
+ argument is explicitly set to None, no landmarks will be drawn.
127
+ connection_drawing_spec: Either a DrawingSpec object or a mapping from hand
128
+ connections to the DrawingSpecs that specifies the connections' drawing
129
+ settings such as color and line thickness. If this argument is explicitly
130
+ set to None, no landmark connections will be drawn.
131
+ Raises:
132
+ ValueError: If one of the followings:
133
+ a) If the input image is not three channel BGR.
134
+ b) If any connetions contain invalid landmark index.
135
+ """
136
+ if not landmark_list:
137
+ return
138
+ if image.shape[2] != _BGR_CHANNELS:
139
+ raise ValueError('Input image must contain three channel bgr data.')
140
+ image_rows, image_cols, _ = image.shape
141
+ idx_to_coordinates = {}
142
+ for idx, landmark in enumerate(landmark_list.landmark):
143
+ if ((landmark.HasField('visibility') and
144
+ landmark.visibility < _VISIBILITY_THRESHOLD) or
145
+ (landmark.HasField('presence') and
146
+ landmark.presence < _PRESENCE_THRESHOLD)):
147
+ continue
148
+ landmark_px = _normalized_to_pixel_coordinates(landmark.x, landmark.y,
149
+ image_cols, image_rows)
150
+ if landmark_px:
151
+ idx_to_coordinates[idx] = landmark_px
152
+ if connections:
153
+ num_landmarks = len(landmark_list.landmark)
154
+ # Draws the connections if the start and end landmarks are both visible.
155
+ for connection in connections:
156
+ start_idx = connection[0]
157
+ end_idx = connection[1]
158
+ if not (0 <= start_idx < num_landmarks and 0 <= end_idx < num_landmarks):
159
+ raise ValueError(f'Landmark index is out of range. Invalid connection '
160
+ f'from landmark #{start_idx} to landmark #{end_idx}.')
161
+ if start_idx in idx_to_coordinates and end_idx in idx_to_coordinates:
162
+ drawing_spec = connection_drawing_spec[connection] if isinstance(
163
+ connection_drawing_spec, Mapping) else connection_drawing_spec
164
+ cv2.line(image, idx_to_coordinates[start_idx],
165
+ idx_to_coordinates[end_idx], drawing_spec.color,
166
+ drawing_spec.thickness)
167
+ # Draws landmark points after finishing the connection lines, which is
168
+ # aesthetically better.
169
+ if landmark_drawing_spec:
170
+ for idx, landmark_px in idx_to_coordinates.items():
171
+ drawing_spec = landmark_drawing_spec[idx] if isinstance(
172
+ landmark_drawing_spec, Mapping) else landmark_drawing_spec
173
+ # White circle border
174
+ circle_border_radius = max(drawing_spec.circle_radius + 1,
175
+ int(drawing_spec.circle_radius * 1.2))
176
+ cv2.circle(image, landmark_px, circle_border_radius, WHITE_COLOR,
177
+ drawing_spec.thickness)
178
+ # Fill color into the circle
179
+ cv2.circle(image, landmark_px, drawing_spec.circle_radius,
180
+ drawing_spec.color, drawing_spec.thickness)
181
+
182
+
183
+ def draw_axis(image: np.ndarray,
184
+ rotation: np.ndarray,
185
+ translation: np.ndarray,
186
+ focal_length: Tuple[float, float] = (1.0, 1.0),
187
+ principal_point: Tuple[float, float] = (0.0, 0.0),
188
+ axis_length: float = 0.1,
189
+ axis_drawing_spec: DrawingSpec = DrawingSpec()):
190
+ """Draws the 3D axis on the image.
191
+ Args:
192
+ image: A three channel BGR image represented as numpy ndarray.
193
+ rotation: Rotation matrix from object to camera coordinate frame.
194
+ translation: Translation vector from object to camera coordinate frame.
195
+ focal_length: camera focal length along x and y directions.
196
+ principal_point: camera principal point in x and y.
197
+ axis_length: length of the axis in the drawing.
198
+ axis_drawing_spec: A DrawingSpec object that specifies the xyz axis drawing
199
+ settings such as line thickness.
200
+ Raises:
201
+ ValueError: If one of the followings:
202
+ a) If the input image is not three channel BGR.
203
+ """
204
+ if image.shape[2] != _BGR_CHANNELS:
205
+ raise ValueError('Input image must contain three channel bgr data.')
206
+ image_rows, image_cols, _ = image.shape
207
+ # Create axis points in camera coordinate frame.
208
+ axis_world = np.float32([[0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1]])
209
+ axis_cam = np.matmul(rotation, axis_length * axis_world.T).T + translation
210
+ x = axis_cam[..., 0]
211
+ y = axis_cam[..., 1]
212
+ z = axis_cam[..., 2]
213
+ # Project 3D points to NDC space.
214
+ fx, fy = focal_length
215
+ px, py = principal_point
216
+ x_ndc = np.clip(-fx * x / (z + 1e-5) + px, -1., 1.)
217
+ y_ndc = np.clip(-fy * y / (z + 1e-5) + py, -1., 1.)
218
+ # Convert from NDC space to image space.
219
+ x_im = np.int32((1 + x_ndc) * 0.5 * image_cols)
220
+ y_im = np.int32((1 - y_ndc) * 0.5 * image_rows)
221
+ # Draw xyz axis on the image.
222
+ origin = (x_im[0], y_im[0])
223
+ x_axis = (x_im[1], y_im[1])
224
+ y_axis = (x_im[2], y_im[2])
225
+ z_axis = (x_im[3], y_im[3])
226
+ cv2.arrowedLine(image, origin, x_axis, RED_COLOR, axis_drawing_spec.thickness)
227
+ cv2.arrowedLine(image, origin, y_axis, GREEN_COLOR,
228
+ axis_drawing_spec.thickness)
229
+ cv2.arrowedLine(image, origin, z_axis, BLUE_COLOR,
230
+ axis_drawing_spec.thickness)
231
+
232
+
233
+ def _normalize_color(color):
234
+ return tuple(v / 255. for v in color)
235
+
236
+
237
+ def plot_landmarks(landmark_list: landmark_pb2.NormalizedLandmarkList,
238
+ connections: Optional[List[Tuple[int, int]]] = None,
239
+ landmark_drawing_spec: DrawingSpec = DrawingSpec(
240
+ color=RED_COLOR, thickness=5),
241
+ connection_drawing_spec: DrawingSpec = DrawingSpec(
242
+ color=BLACK_COLOR, thickness=5),
243
+ elevation: int = 10,
244
+ azimuth: int = 10):
245
+ """Plot the landmarks and the connections in matplotlib 3d.
246
+ Args:
247
+ landmark_list: A normalized landmark list proto message to be plotted.
248
+ connections: A list of landmark index tuples that specifies how landmarks to
249
+ be connected.
250
+ landmark_drawing_spec: A DrawingSpec object that specifies the landmarks'
251
+ drawing settings such as color and line thickness.
252
+ connection_drawing_spec: A DrawingSpec object that specifies the
253
+ connections' drawing settings such as color and line thickness.
254
+ elevation: The elevation from which to view the plot.
255
+ azimuth: the azimuth angle to rotate the plot.
256
+ Raises:
257
+ ValueError: If any connection contains an invalid landmark index.
258
+ """
259
+ if not landmark_list:
260
+ return
261
+ plt.figure(figsize=(10, 10))
262
+ ax = plt.axes(projection='3d')
263
+ ax.view_init(elev=elevation, azim=azimuth)
264
+ plotted_landmarks = {}
265
+ for idx, landmark in enumerate(landmark_list.landmark):
266
+ if ((landmark.HasField('visibility') and
267
+ landmark.visibility < _VISIBILITY_THRESHOLD) or
268
+ (landmark.HasField('presence') and
269
+ landmark.presence < _PRESENCE_THRESHOLD)):
270
+ continue
271
+ ax.scatter3D(
272
+ xs=[-landmark.z],
273
+ ys=[landmark.x],
274
+ zs=[-landmark.y],
275
+ color=_normalize_color(landmark_drawing_spec.color[::-1]),
276
+ linewidth=landmark_drawing_spec.thickness)
277
+ plotted_landmarks[idx] = (-landmark.z, landmark.x, -landmark.y)
278
+ if connections:
279
+ num_landmarks = len(landmark_list.landmark)
280
+ # Draws the connections if the start and end landmarks are both visible.
281
+ for connection in connections:
282
+ start_idx = connection[0]
283
+ end_idx = connection[1]
284
+ if not (0 <= start_idx < num_landmarks and 0 <= end_idx < num_landmarks):
285
+ raise ValueError(f'Landmark index is out of range. Invalid connection '
286
+ f'from landmark #{start_idx} to landmark #{end_idx}.')
287
+ if start_idx in plotted_landmarks and end_idx in plotted_landmarks:
288
+ landmark_pair = [
289
+ plotted_landmarks[start_idx], plotted_landmarks[end_idx]
290
+ ]
291
+ ax.plot3D(
292
+ xs=[landmark_pair[0][0], landmark_pair[1][0]],
293
+ ys=[landmark_pair[0][1], landmark_pair[1][1]],
294
+ zs=[landmark_pair[0][2], landmark_pair[1][2]],
295
+ color=_normalize_color(connection_drawing_spec.color[::-1]),
296
+ linewidth=connection_drawing_spec.thickness)
297
+ plt.show()