Download smirk/src/FLAME/FLAME.py from vegagoto123/UIKA: direct link, hf CLI and curl.
- Browser
- Download file 15.4 kB
-
https://huggingface.co/vegagoto123/UIKA/resolve/main/smirk/src/FLAME/FLAME.py
- Command line
-
hf download hf://vegagoto123/UIKA/smirk/src/FLAME/FLAME.py
-
curl -L -o FLAME.py https://huggingface.co/vegagoto123/UIKA/resolve/main/smirk/src/FLAME/FLAME.py
15.4 kB
| # -*- coding: utf-8 -*- | |
| # | |
| # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is | |
| # holder of all proprietary rights on this computer program. | |
| # Using this computer program means that you agree to the terms | |
| # in the LICENSE file included with this software distribution. | |
| # Any use not explicitly granted by the LICENSE is prohibited. | |
| # | |
| # Copyright©2019 Max-Planck-Gesellschaft zur Förderung | |
| # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute | |
| # for Intelligent Systems. All rights reserved. | |
| # | |
| # For comments or questions, please email us at deca@tue.mpg.de | |
| # For commercial licensing contact, please contact ps-license@tuebingen.mpg.de | |
| import torch | |
| import torch.nn as nn | |
| import numpy as np | |
| np.bool = np.bool_ | |
| np.int = np.int64 | |
| np.float = np.float64 | |
| np.complex = np.complex128 | |
| np.object = np.object_ | |
| np.unicode = np.str_ | |
| np.str = np.str_ | |
| import pickle | |
| import torch.nn.functional as F | |
| from .lbs import lbs, batch_rodrigues, vertices2landmarks, rot_mat_to_euler | |
| def to_tensor(array, dtype=torch.float32): | |
| if 'torch.tensor' not in str(type(array)): | |
| return torch.tensor(array, dtype=dtype) | |
| def to_np(array, dtype=np.float32): | |
| if 'scipy.sparse' in str(type(array)): | |
| array = array.todense() | |
| return np.array(array, dtype=dtype) | |
| class Struct(object): | |
| def __init__(self, **kwargs): | |
| for key, val in kwargs.items(): | |
| setattr(self, key, val) | |
| class FLAME(nn.Module): | |
| """ | |
| borrowed from https://github.com/soubhiksanyal/FLAME_PyTorch/blob/master/FLAME.py | |
| Given flame parameters this class generates a differentiable FLAME function | |
| which outputs the a mesh and 2D/3D facial landmarks | |
| """ | |
| def __init__(self, flame_model_path='assets/FLAME2020/generic_model.pkl', | |
| flame_lmk_embedding_path='assets/landmark_embedding.npy', n_shape=300, n_exp=50): | |
| super(FLAME, self).__init__() | |
| with open(flame_model_path, 'rb') as f: | |
| ss = pickle.load(f, encoding='latin1') | |
| flame_model = Struct(**ss) | |
| self.n_shape = n_shape | |
| self.n_exp = n_exp | |
| self.dtype = torch.float32 | |
| self.register_buffer('faces_tensor', to_tensor(to_np(flame_model.f, dtype=np.int64), dtype=torch.long)) | |
| # The vertices of the template model | |
| print('Using generic FLAME model') | |
| self.register_buffer('v_template', to_tensor(to_np(flame_model.v_template), dtype=self.dtype)) | |
| # The shape components and expression | |
| shapedirs = to_tensor(to_np(flame_model.shapedirs), dtype=self.dtype) | |
| shapedirs = torch.cat([shapedirs[:,:,:n_shape], shapedirs[:,:,300:300+n_exp]], 2) | |
| self.register_buffer('shapedirs', shapedirs) | |
| # The pose components | |
| num_pose_basis = flame_model.posedirs.shape[-1] | |
| posedirs = np.reshape(flame_model.posedirs, [-1, num_pose_basis]).T | |
| self.register_buffer('posedirs', to_tensor(to_np(posedirs), dtype=self.dtype)) | |
| # | |
| self.register_buffer('J_regressor', to_tensor(to_np(flame_model.J_regressor), dtype=self.dtype)) | |
| parents = to_tensor(to_np(flame_model.kintree_table[0])).long(); parents[0] = -1 | |
| self.register_buffer('parents', parents) | |
| self.register_buffer('lbs_weights', to_tensor(to_np(flame_model.weights), dtype=self.dtype)) | |
| self.register_buffer('l_eyelid', torch.from_numpy(np.load(f'assets/l_eyelid.npy')).to(self.dtype)[None]) | |
| self.register_buffer('r_eyelid', torch.from_numpy(np.load(f'assets/r_eyelid.npy')).to(self.dtype)[None]) | |
| # Fixing Eyeball and neck rotation | |
| default_eyball_pose = torch.zeros([1, 6], dtype=self.dtype, requires_grad=False) | |
| self.register_parameter('eye_pose', nn.Parameter(default_eyball_pose, | |
| requires_grad=False)) | |
| default_neck_pose = torch.zeros([1, 3], dtype=self.dtype, requires_grad=False) | |
| self.register_parameter('neck_pose', nn.Parameter(default_neck_pose, | |
| requires_grad=False)) | |
| # Static and Dynamic Landmark embeddings for FLAME | |
| lmk_embeddings = np.load(flame_lmk_embedding_path, allow_pickle=True, encoding='latin1') | |
| lmk_embeddings = lmk_embeddings[()] | |
| self.register_buffer('lmk_faces_idx', torch.from_numpy(lmk_embeddings['static_lmk_faces_idx']).long()) | |
| self.register_buffer('lmk_bary_coords', torch.from_numpy(lmk_embeddings['static_lmk_bary_coords']).to(self.dtype)) | |
| self.register_buffer('dynamic_lmk_faces_idx', lmk_embeddings['dynamic_lmk_faces_idx'].long()) | |
| self.register_buffer('dynamic_lmk_bary_coords', lmk_embeddings['dynamic_lmk_bary_coords'].to(self.dtype)) | |
| self.register_buffer('full_lmk_faces_idx', torch.from_numpy(lmk_embeddings['full_lmk_faces_idx']).long()) | |
| self.register_buffer('full_lmk_bary_coords', torch.from_numpy(lmk_embeddings['full_lmk_bary_coords']).to(self.dtype)) | |
| neck_kin_chain = []; NECK_IDX=1 | |
| curr_idx = torch.tensor(NECK_IDX, dtype=torch.long) | |
| while curr_idx != -1: | |
| neck_kin_chain.append(curr_idx) | |
| curr_idx = self.parents[curr_idx] | |
| self.register_buffer('neck_kin_chain', torch.stack(neck_kin_chain)) | |
| lmk_embeddings_mp = np.load("assets/mediapipe_landmark_embedding/mediapipe_landmark_embedding.npz") | |
| self.register_buffer('mp_lmk_faces_idx', torch.from_numpy(lmk_embeddings_mp['lmk_face_idx'].astype('int32')).long()) | |
| self.register_buffer('mp_lmk_bary_coords', torch.from_numpy(lmk_embeddings_mp['lmk_b_coords']).to(self.dtype)) | |
| def _find_dynamic_lmk_idx_and_bcoords(self, pose, dynamic_lmk_faces_idx, | |
| dynamic_lmk_b_coords, | |
| neck_kin_chain, dtype=torch.float32): | |
| """ | |
| Selects the face contour depending on the reletive position of the head | |
| Input: | |
| vertices: N X num_of_vertices X 3 | |
| pose: N X full pose | |
| dynamic_lmk_faces_idx: The list of contour face indexes | |
| dynamic_lmk_b_coords: The list of contour barycentric weights | |
| neck_kin_chain: The tree to consider for the relative rotation | |
| dtype: Data type | |
| return: | |
| The contour face indexes and the corresponding barycentric weights | |
| """ | |
| batch_size = pose.shape[0] | |
| aa_pose = torch.index_select(pose.view(batch_size, -1, 3), 1, | |
| neck_kin_chain) | |
| rot_mats = batch_rodrigues( | |
| aa_pose.view(-1, 3), dtype=dtype).view(batch_size, -1, 3, 3) | |
| rel_rot_mat = torch.eye(3, device=pose.device, | |
| dtype=dtype).unsqueeze_(dim=0).expand(batch_size, -1, -1) | |
| for idx in range(len(neck_kin_chain)): | |
| rel_rot_mat = torch.bmm(rot_mats[:, idx], rel_rot_mat) | |
| y_rot_angle = torch.round( | |
| torch.clamp(rot_mat_to_euler(rel_rot_mat) * 180.0 / np.pi, | |
| max=39)).to(dtype=torch.long) | |
| neg_mask = y_rot_angle.lt(0).to(dtype=torch.long) | |
| mask = y_rot_angle.lt(-39).to(dtype=torch.long) | |
| neg_vals = mask * 78 + (1 - mask) * (39 - y_rot_angle) | |
| y_rot_angle = (neg_mask * neg_vals + | |
| (1 - neg_mask) * y_rot_angle) | |
| dyn_lmk_faces_idx = torch.index_select(dynamic_lmk_faces_idx, | |
| 0, y_rot_angle) | |
| dyn_lmk_b_coords = torch.index_select(dynamic_lmk_b_coords, | |
| 0, y_rot_angle) | |
| return dyn_lmk_faces_idx, dyn_lmk_b_coords | |
| def _vertices2landmarks(self, vertices, faces, lmk_faces_idx, lmk_bary_coords): | |
| """ | |
| Calculates landmarks by barycentric interpolation | |
| Input: | |
| vertices: torch.tensor NxVx3, dtype = torch.float32 | |
| The tensor of input vertices | |
| faces: torch.tensor (N*F)x3, dtype = torch.long | |
| The faces of the mesh | |
| lmk_faces_idx: torch.tensor N X L, dtype = torch.long | |
| The tensor with the indices of the faces used to calculate the | |
| landmarks. | |
| lmk_bary_coords: torch.tensor N X L X 3, dtype = torch.float32 | |
| The tensor of barycentric coordinates that are used to interpolate | |
| the landmarks | |
| Returns: | |
| landmarks: torch.tensor NxLx3, dtype = torch.float32 | |
| The coordinates of the landmarks for each mesh in the batch | |
| """ | |
| # Extract the indices of the vertices for each face | |
| # NxLx3 | |
| batch_size, num_verts = vertices.shape[:dd2] | |
| lmk_faces = torch.index_select(faces, 0, lmk_faces_idx.view(-1)).view( | |
| 1, -1, 3).view(batch_size, lmk_faces_idx.shape[1], -1) | |
| lmk_faces += torch.arange(batch_size, dtype=torch.long).view(-1, 1, 1).to( | |
| device=vertices.device) * num_verts | |
| lmk_vertices = vertices.view(-1, 3)[lmk_faces] | |
| landmarks = torch.einsum('blfi,blf->bli', [lmk_vertices, lmk_bary_coords]) | |
| return landmarks | |
| def seletec_3d68(self, vertices): | |
| landmarks3d = vertices2landmarks(vertices, self.faces_tensor, | |
| self.full_lmk_faces_idx.repeat(vertices.shape[0], 1), | |
| self.full_lmk_bary_coords.repeat(vertices.shape[0], 1, 1)) | |
| return landmarks3d | |
| def get_landmarks(self, vertices): | |
| """ | |
| Input: | |
| shape_params: N X number of shape parameters | |
| expression_params: N X number of expression parameters | |
| pose_params: N X number of pose parameters (6) | |
| return:d | |
| vertices: N X V X 3 | |
| landmarks: N X number of landmarks X 3 | |
| """ | |
| batch_size = vertices.shape[0] | |
| template_vertices = self.v_template.unsqueeze(0).expand(batch_size, -1, -1) | |
| lmk_faces_idx = self.lmk_faces_idx.unsqueeze(dim=0).expand(batch_size, -1) | |
| lmk_bary_coords = self.lmk_bary_coords.unsqueeze(dim=0).expand(batch_size, -1, -1) | |
| dyn_lmk_faces_idx, dyn_lmk_bary_coords = self._find_dynamic_lmk_idx_and_bcoords( | |
| full_pose, self.dynamic_lmk_faces_idx, | |
| self.dynamic_lmk_bary_coords, | |
| self.neck_kin_chain, dtype=self.dtype) | |
| lmk_faces_idx = torch.cat([dyn_lmk_faces_idx, lmk_faces_idx], 1) | |
| lmk_bary_coords = torch.cat([dyn_lmk_bary_coords, lmk_bary_coords], 1) | |
| landmarks2d = vertices2landmarks(vertices, self.faces_tensor, | |
| lmk_faces_idx, | |
| lmk_bary_coords) | |
| bz = vertices.shape[0] | |
| landmarks3d = vertices2landmarks(vertices, self.faces_tensor, | |
| self.full_lmk_faces_idx.repeat(bz, 1), | |
| self.full_lmk_bary_coords.repeat(bz, 1, 1)) | |
| return vertices, landmarks2d, landmarks3d | |
| def forward(self, param_dictionary, zero_expression=False, zero_shape=False, zero_pose=False): | |
| shape_params = param_dictionary['shape_params'] | |
| expression_params = param_dictionary['expression_params'] | |
| pose_params = param_dictionary.get('pose_params', None) | |
| jaw_params = param_dictionary.get('jaw_params', None) | |
| eye_pose_params = param_dictionary.get('eye_pose_params', None) | |
| neck_pose_params = param_dictionary.get('neck_pose_params', None) | |
| eyelid_params = param_dictionary.get('eyelid_params', None) | |
| batch_size = shape_params.shape[0] | |
| # Adjust expression params size if needed | |
| if expression_params.shape[1] < self.n_exp: | |
| expression_params = torch.cat([expression_params, torch.zeros(expression_params.shape[0], self.n_exp - expression_params.shape[1]).to(shape_params.device)], dim=1) | |
| if shape_params.shape[1] < self.n_shape: | |
| shape_params = torch.cat([shape_params, torch.zeros(shape_params.shape[0], self.n_shape - shape_params.shape[1]).to(shape_params.device)], dim=1) | |
| # Zero out the expression and pose parameters if needed | |
| if zero_expression: | |
| expression_params = torch.zeros_like(expression_params).to(shape_params.device) | |
| jaw_params = torch.zeros_like(jaw_params).to(shape_params.device) | |
| if zero_shape: | |
| shape_params = torch.zeros_like(shape_params).to(shape_params.device) | |
| if zero_pose: | |
| pose_params = torch.zeros_like(pose_params).to(shape_params.device) | |
| pose_params[...,0] = 0.2 | |
| pose_params[...,1] = -0.7 | |
| if pose_params is None: | |
| pose_params = self.pose_params.expand(batch_size, -1) | |
| if eye_pose_params is None: | |
| eye_pose_params = self.eye_pose.expand(batch_size, -1) | |
| if neck_pose_params is None: | |
| neck_pose_params = self.neck_pose.expand(batch_size, -1) | |
| betas = torch.cat([shape_params, expression_params], dim=1) | |
| full_pose = torch.cat([pose_params, neck_pose_params, jaw_params, eye_pose_params], dim=1) | |
| template_vertices = self.v_template.unsqueeze(0).expand(batch_size, -1, -1) | |
| vertices, _ = lbs(betas, full_pose, template_vertices, | |
| self.shapedirs, self.posedirs, | |
| self.J_regressor, self.parents, | |
| self.lbs_weights, dtype=self.dtype) | |
| if eyelid_params is not None: | |
| vertices = vertices + self.r_eyelid.expand(batch_size, -1, -1) * eyelid_params[:, 1:2, None] | |
| vertices = vertices + self.l_eyelid.expand(batch_size, -1, -1) * eyelid_params[:, 0:1, None] | |
| lmk_faces_idx = self.lmk_faces_idx.unsqueeze(dim=0).expand(batch_size, -1) | |
| lmk_bary_coords = self.lmk_bary_coords.unsqueeze(dim=0).expand(batch_size, -1, -1) | |
| dyn_lmk_faces_idx, dyn_lmk_bary_coords = self._find_dynamic_lmk_idx_and_bcoords( | |
| full_pose, self.dynamic_lmk_faces_idx, | |
| self.dynamic_lmk_bary_coords, | |
| self.neck_kin_chain, dtype=self.dtype) | |
| lmk_faces_idx = torch.cat([dyn_lmk_faces_idx, lmk_faces_idx], 1) | |
| lmk_bary_coords = torch.cat([dyn_lmk_bary_coords, lmk_bary_coords], 1) | |
| landmarks2d = vertices2landmarks(vertices, self.faces_tensor, | |
| lmk_faces_idx, | |
| lmk_bary_coords) | |
| bz = vertices.shape[0] | |
| landmarks3d = vertices2landmarks(vertices, self.faces_tensor, | |
| self.full_lmk_faces_idx.repeat(bz, 1), | |
| self.full_lmk_bary_coords.repeat(bz, 1, 1)) | |
| landmarksmp = vertices2landmarks(vertices, self.faces_tensor, | |
| self.mp_lmk_faces_idx.repeat(vertices.shape[0], 1), | |
| self.mp_lmk_bary_coords.repeat(vertices.shape[0], 1, 1)) | |
| return { | |
| 'vertices': vertices, | |
| 'landmarks_fan': landmarks2d, | |
| 'landmarks_fan_3d': landmarks3d, | |
| 'landmarks_mp': landmarksmp | |
| } | |