File size: 7,287 Bytes
d603ee0
773026d
d603ee0
 
 
 
 
 
 
 
 
 
 
 
aef4429
 
 
 
d603ee0
 
aef4429
46f9ac4
d603ee0
aef4429
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d603ee0
aef4429
 
 
773026d
 
aef4429
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
import cv2
import pickle
import os
import argparse
import mediapipe as mp
import numpy as np
import glob
import time
import yaml
from PIL import Image

import streamlit as st
st.set_page_config(layout="wide")

from utils.drawing_utils import draw_all_informations
from utils.general import resize_img, get_cache_informations, parse_head_pose_informations
from utils.streamlit_options import default_UI
from utils.detection import detect_face_pose_informations_from_image


IMG_SUFFIX = ['jpeg', 'jpg', 'png']


def main():
    parser = argparse.ArgumentParser()
    # Pose position
    parser.add_argument('--head_pose_info', default='configs.yaml', help='path to head pose information')

    args = parser.parse_args()

    ## Streamlit options
    st.title("Head pose estimation tracking app 📷")
    left_col, right_col = st.columns([2, 6])

    with left_col:
        default_UI()

    img_size = st.session_state.img_size.split('x')
    img_size = [int(s) for s in img_size]


    if st.session_state.img_upload is not None:
        mp_face_mesh = mp.solutions.face_mesh
        mp_face_detection = mp.solutions.face_detection
        face_mesh = mp_face_mesh.FaceMesh(min_detection_confidence=.5, min_tracking_confidence=0.5)

        with open(args.head_pose_info, 'r') as f:
            head_pose_info = yaml.load(f, Loader=yaml.FullLoader)

        parse_head_pose_informations(head_pose_info, st.session_state)

        img_dict = {}  # List of images upload in RGB
        save_dict = {'img_size': img_size, 
                    'position_horizontal_thresholds': st.session_state.position_horizontal_thresholds,
                    'position_vertical_thresholds': st.session_state.position_vertical_thresholds}
        image_results = {}  # List of results for each image in RGB
        image_names = []  # List image names 
        face_detection_dict = {}  # Face detection information of each image
        face_pose_dict = {}  # Face pose information of each image
        chosen_rectangle_pos_dict = {}  # Positions where have face for each image
        face_direction_dict = {}  # direction of each face for each image
        face_position_dict = {}  # position of each face for each image
        face_coordinate_dict = {}  # coordinate of each face for each image
        face_area_dict = {}
        
        if st.session_state.using_local_cache:
            cache_dict = get_cache_informations(save_dict)
        else:
            cache_dict = {}

        for img_upload_file in st.session_state.img_upload:
            image = np.array(Image.open(img_upload_file))
            if len(image.shape) > 2 and image.shape[2] == 4:
                image = cv2.cvtColor(image, cv2.COLOR_RGBA2RGB)

            image_name = img_upload_file.name.split('.')[0]

            image_names.append(image_name)
            img_dict[image_name] = image

            if image_name in cache_dict:
                face_detection_dict[image_name] = cache_dict[image_name]['face_detection']
                face_pose_dict[image_name] = cache_dict[image_name]['face_pose']
                chosen_rectangle_pos_dict[image_name] = cache_dict[image_name]['chosen_rectangle_pos']
                face_direction_dict[image_name] = cache_dict[image_name]['face_direction']
                face_position_dict[image_name] = cache_dict[image_name]['face_position']
                face_coordinate_dict[image_name] = cache_dict[image_name]['face_coordinate']
                face_area_dict[image_name] = cache_dict[image_name]['face_area']
        total_time = 0

        with mp_face_detection.FaceDetection(
            model_selection=1, min_detection_confidence=0.5) as face_detection:
            for id, (image_name, image) in enumerate(img_dict.items()):
                if isinstance(image, str):
                    image = cv2.imread(image)

                # Check if the image has only 3 channels
                if image.shape[-1] == 1:
                    image = np.stack([image]*3, axis=-1)

                start = time.time()

                image, h_ratio, w_ratio, pad_top, pad_bot, pad_left, pad_right, img_size_no_pad = \
                                resize_img(image, img_size, return_all_infos=True)
                image_results[image_name] = image

                # To improve performance
                image.flags.writeable = False

                if not image_name in cache_dict:
                    face_detection_infos, face_direction_infos, face_position_infos, face_coordinate_infos, face_pose_infos, face_area_infos, chosen_rectangle_pos_list = \
                                detect_face_pose_informations_from_image(image, face_detection, face_mesh, img_size, (pad_top, pad_bot, pad_left, pad_right), img_size_no_pad, head_pose_info)

                    face_detection_dict[image_name] = face_detection_infos
                    face_direction_dict[image_name] = face_direction_infos
                    face_position_dict[image_name] = face_position_infos
                    face_coordinate_dict[image_name] = face_coordinate_infos
                    face_pose_dict[image_name] = face_pose_infos
                    chosen_rectangle_pos_dict[image_name] = chosen_rectangle_pos_list
                    face_area_dict[image_name] = face_area_infos

                end = time.time()
                cur_time = end - start
                total_time += cur_time

                draw_all_informations(image_results[image_name], face_detection_dict[image_name], face_direction_dict[image_name], face_position_dict[image_name],
                                    face_coordinate_dict[image_name], face_pose_dict[image_name], face_area_dict[image_name], chosen_rectangle_pos_dict[image_name],
                                    img_size, (pad_top, pad_bot, pad_left, pad_right), (w_ratio, h_ratio), st.session_state, head_pose_info)

                ## Save results to dict
                if st.session_state.using_local_cache:
                    save_dict[image_name] = {}
                    save_dict[image_name]['face_detection'] = face_detection_dict[image_name]
                    save_dict[image_name]['face_pose'] = face_pose_dict[image_name]
                    save_dict[image_name]['face_direction'] = face_direction_dict[image_name]
                    save_dict[image_name]['face_position'] = face_position_dict[image_name]
                    save_dict[image_name]['chosen_rectangle_pos'] = chosen_rectangle_pos_dict[image_name]
                    save_dict[image_name]['face_coordinate'] = face_coordinate_dict[image_name]
                    save_dict[image_name]['face_area'] = face_area_dict[image_name]

                    if image_name in cache_dict:  # Remove image from cache to save memory
                        del cache_dict[image_name]

        print(f"FPS: {len(image_results) / total_time}")
    ## Save results to local cache
    if st.session_state.using_local_cache:
        with open('cache.pkl', 'wb') as f:
            pickle.dump(save_dict, f, protocol=pickle.HIGHEST_PROTOCOL)

    image_results = [image_results[image_name] for image_name in image_names]
    with right_col:
        st.image(image_results, width=st.session_state.width_visual, caption=image_names)


if __name__ == '__main__':
    main()