Download src/deform_model.py from vegagoto123/gha: direct link, hf CLI and curl.
- Browser
- Download file 35.6 kB
-
https://huggingface.co/vegagoto123/gha/resolve/main/src/deform_model.py
- Command line
-
hf download hf://vegagoto123/gha/src/deform_model.py
-
curl -L -o deform_model.py https://huggingface.co/vegagoto123/gha/resolve/main/src/deform_model.py
35.6 kB
| import sys | |
| import pickle | |
| import torch | |
| import numpy as np | |
| from torch import nn | |
| import math | |
| import torch.nn.functional as F | |
| from pytorch3d.io import load_obj | |
| import torchvision | |
| import os | |
| from flame import FLAME_mica, parse_args | |
| from utils.general_utils import Pytorch3dRasterizer, Embedder, a_in_b_torch, face_vertices_gen | |
| from model_training.model.flame import FLAMELayer, FLAME_CONSTS, FlameParams | |
| import time | |
| from dataclasses import replace | |
| def rot_mat_from_6dof(v: torch.Tensor) -> torch.Tensor: | |
| assert v.shape[-1] == 6 | |
| v = v.view(-1, 6) | |
| vx, vy = v[..., :3].clone(), v[..., 3:].clone() | |
| b1 = F.normalize(vx, dim=-1) | |
| b3 = F.normalize(torch.cross(b1, vy), dim=-1) | |
| b2 = - torch.cross(b1, b3) | |
| return torch.stack((b1, b2, b3), dim=-1) | |
| class Deform_Model(nn.Module): | |
| def __init__(self, device): | |
| super().__init__() | |
| self.device = device | |
| self.batch_size = 1 | |
| self.flame_constants = FLAME_CONSTS | |
| mica_flame_config = parse_args() | |
| # MICA预测 表情参数 目前使用的是dad3dnet预测 | |
| # self.flame_model = FLAME_mica(mica_flame_config).to(self.device) | |
| self.default_shape_code = torch.zeros(1, 300) | |
| self.default_expr_code = torch.zeros(1, 100) | |
| # positional encoding | |
| self.pts_freq = 8 | |
| self.pts_embedder = Embedder(self.pts_freq) | |
| # verts[5023, 3] | |
| verts, faces, aux = load_obj('flame/FlameMesh.obj', load_textures=False) | |
| # faces 4 | |
| # faces.verts_idx # (10006, 3),每个三角面对应 verts 里的 3 个顶点索引 | |
| # faces.textures_idx # (10006, 3),每个三角面对应 aux.verts_uvs 里的 3 个 UV 索引 | |
| # faces.normals_idx # (10006, 3),每个三角面对应 aux.normals 的法线索引 | |
| # faces.materials_idx # (10006, ), 每个 face 对应的材质索引 | |
| # aux | |
| # aux.verts_uvs # (5150, 2),UV 坐标 | |
| # aux.normals # (9317, 3),法线 | |
| # aux.material_colors # None 材质颜色 | |
| # aux.texture_images # None 贴图图片 | |
| # aux.texture_atlas # None texture atlas | |
| uv_coords = aux.verts_uvs[None, ...] # (1, 5150, 2) | |
| uv_coords = uv_coords * 2 - 1 | |
| uv_coords[..., 1] = - uv_coords[..., 1] | |
| self.uvcoords = torch.cat([uv_coords, uv_coords[:, :, 0:1] * 0. + 1.], -1).to(self.device) | |
| self.uvfaces = faces.textures_idx[None, ...].to(self.device) # (1, 10006, 3) | |
| self.tri_faces = faces.verts_idx[None, ...].to(self.device) # (1, 10006, 3) | |
| # rasterizer | |
| self.uv_size = 512 | |
| self.uv_rasterizer = Pytorch3dRasterizer(self.uv_size) | |
| # flame mask | |
| full_id = np.array(range(5023)).astype(int) | |
| neckhead_id_list = list(set(full_id)) | |
| self.neckhead_id_list = neckhead_id_list | |
| self.neckhead_id_tensor = torch.tensor(self.neckhead_id_list, dtype=torch.int64).to(self.device) | |
| # 新增:光栅化缓存标志和缓存数据 | |
| self.rasterization_cached = False | |
| self.cached_pix_to_face_ori = None # (237544, 512, 512, 1) 表示 UV 图上每个像素落在哪个三角面上 | |
| self.cached_bary_coords = None # (B, 512, 512, 1, 3) 每个有效像素在对应三角面内的重心坐标权重,后面用它把 UV 像素插值回 3D 面片上的点。 | |
| self.cached_uvmask_flaten_idx = None # (512 * 512) 缓存有效 UV 像素 mask | |
| self.cached_pix_to_face = None # (N_valid,) 缓存只保留有效 UV 像素后的 face index 每个有效 UV 像素对应哪个三角面 | |
| self.cached_pix_to_v_idx = None # (N_valid, 3) 缓存每个有效 UV 像素对应三角面的 3 个 3D 顶点索引 | |
| self.cached_uv_vertices_shape = None # (1, N_valid, 3) 缓存 canonical/default shape 下,每个有效 UV 像素插值出来的 3D 点。 | |
| self.cached_uv_vertices_shape_embeded = None # (1, N_valid, embed_dim) 缓存对 uv_vertices_shape 做 positional encoding 后的结果 | |
| self.cached_v_num = None # 缓存有效点数量 N_valid | |
| self.cached_uv_head_idx = None # (N_valid,) 缓存最终用于筛选点的 mask | |
| # 同时清除FLAME缓存 | |
| self.cached_flame_geometry = None | |
| self.cached_flame_params = None | |
| self.init_networks() | |
| def init_networks(self): | |
| ## full mica | |
| self.deformNet = MLP( | |
| input_dim=self.pts_embedder.dim_embeded+100+64,#+64,#74 | |
| output_dim=10, | |
| hidden_dim=256, | |
| hidden_layers=6 | |
| ) | |
| def example_init(self, codedict, flame): | |
| # speed up | |
| # print(f"[DEBUG] example_init called with codedict shape: {codedict.shape if hasattr(codedict, 'shape') else 'N/A'}", flush=True) | |
| # 检查是否已经缓存了光栅化结果 | |
| if self.rasterization_cached: | |
| # print(f"[INFO] Using cached rasterization results - skipping expensive UV rasterization", flush=True) | |
| # 恢复缓存的光栅化结果 | |
| self.pix_to_face_ori = self.cached_pix_to_face_ori | |
| self.bary_coords = self.cached_bary_coords | |
| self.uvmask_flaten_idx = self.cached_uvmask_flaten_idx | |
| self.pix_to_face = self.cached_pix_to_face | |
| self.pix_to_v_idx = self.cached_pix_to_v_idx | |
| self.uv_vertices_shape = self.cached_uv_vertices_shape | |
| self.uv_vertices_shape_embeded = self.cached_uv_vertices_shape_embeded | |
| self.v_num = self.cached_v_num | |
| self.uv_head_idx = self.cached_uv_head_idx | |
| # print(f"[INFO] Cached rasterization results restored successfully", flush=True) | |
| return | |
| # print(f"[INFO] First time execution - performing full UV rasterization", flush=True) | |
| # 总体计时开始 | |
| # torch.cuda.synchronize() | |
| # total_start_time = time.time() | |
| shape_code = codedict.shape.detach() | |
| batch_size = shape_code.shape[0] | |
| # 步骤1: FLAME模型前向传播 | |
| # torch.cuda.synchronize() | |
| # step1_start = time.time() | |
| # 深引用版本(新) | |
| temp_codedict = replace(codedict, expression=self.default_expr_code) | |
| # print(f"[DEBUG] About to call flame.forward", flush=True) | |
| geometry_shape = flame.forward(temp_codedict, zero_rot=True).cuda() | |
| # 浅引用版本(原来的) | |
| # codedict.expression = self.default_expr_code | |
| # # print(f"[DEBUG] About to call flame.forward", flush=True) | |
| # geometry_shape = flame.forward(codedict, zero_rot=True).cuda() | |
| # torch.cuda.synchronize() | |
| # step1_time = time.time() - step1_start | |
| # print(f"[TIMING] Step 1 - FLAME forward: {step1_time:.4f} seconds", flush=True) | |
| # print(f"[DEBUG] flame.forward completed, geometry_shape shape: {geometry_shape.shape}", flush=True) | |
| # 步骤2: 生成面部顶点 | |
| # torch.cuda.synchronize() | |
| # step2_start = time.time() | |
| # self.tri_faces是flame 的三角面索引 | |
| # 得到每个三角面片的顶点索引 | |
| face_vertices_shape = face_vertices_gen(geometry_shape, self.tri_faces.expand(batch_size, -1, -1)) | |
| # torch.cuda.synchronize() | |
| # step2_time = time.time() - step2_start | |
| # print(f"[TIMING] Step 2 - Face vertices generation: {step2_time:.4f} seconds", flush=True) | |
| # print(f"[DEBUG] face_vertices_shape created, shape: {face_vertices_shape.shape}", flush=True) | |
| # 步骤3: UV光栅化 | |
| # torch.cuda.synchronize() | |
| # step3_start = time.time() | |
| rast_out, pix_to_face, bary_coords = self.uv_rasterizer(self.uvcoords.expand(batch_size, -1, -1), | |
| self.uvfaces.expand(batch_size, -1, -1), | |
| face_vertices_shape) | |
| # torch.cuda.synchronize() | |
| # step3_time = time.time() - step3_start | |
| # print(f"[TIMING] Step 3 - UV rasterization: {step3_time:.4f} seconds", flush=True) | |
| # print(f"[DEBUG] UV rasterization completed", flush=True) | |
| # 步骤4: 数据预处理和索引计算 | |
| # torch.cuda.synchronize() | |
| # step4_start = time.time() | |
| self.pix_to_face_ori = pix_to_face | |
| self.bary_coords = bary_coords | |
| uvmask = rast_out[:, -1].unsqueeze(1) | |
| uvmask_flaten = uvmask[0].view(uvmask.shape[1], -1).permute(1, 0).squeeze(1) # batch=1 | |
| self.uvmask_flaten_idx = (uvmask_flaten[:]>0) | |
| pix_to_face_flaten = pix_to_face[0].clone().view(-1) # batch=1 | |
| self.pix_to_face = pix_to_face_flaten[self.uvmask_flaten_idx] # pix to face idx | |
| self.pix_to_v_idx = self.tri_faces[0, self.pix_to_face, :] # pix to vert idx | |
| uv_vertices_shape = rast_out[:, :3] | |
| uv_vertices_shape_flaten = uv_vertices_shape[0].view(uv_vertices_shape.shape[1], -1).permute(1, 0) # batch=1 | |
| uv_vertices_shape = uv_vertices_shape_flaten[self.uvmask_flaten_idx].unsqueeze(0) | |
| self.uv_vertices_shape = uv_vertices_shape | |
| self.uv_vertices_shape_embeded = self.pts_embedder(self.uv_vertices_shape) | |
| self.v_num = self.uv_vertices_shape_embeded.shape[1] | |
| # mask | |
| self.uv_head_idx = ( | |
| a_in_b_torch(self.pix_to_v_idx[:,0], self.neckhead_id_tensor) | |
| & a_in_b_torch(self.pix_to_v_idx[:,1], self.neckhead_id_tensor) | |
| & a_in_b_torch(self.pix_to_v_idx[:,2], self.neckhead_id_tensor) | |
| ) | |
| # 缓存光栅化结果 | |
| self.cached_pix_to_face_ori = self.pix_to_face_ori.clone() | |
| self.cached_bary_coords = self.bary_coords.clone() | |
| self.cached_uvmask_flaten_idx = self.uvmask_flaten_idx.clone() | |
| self.cached_pix_to_face = self.pix_to_face.clone() | |
| self.cached_pix_to_v_idx = self.pix_to_v_idx.clone() | |
| self.cached_uv_vertices_shape = self.uv_vertices_shape.clone() | |
| self.cached_uv_vertices_shape_embeded = self.uv_vertices_shape_embeded.clone() | |
| self.cached_v_num = self.v_num | |
| self.cached_uv_head_idx = self.uv_head_idx.clone() | |
| # 标记光栅化已完成缓存 | |
| self.rasterization_cached = True | |
| # print(f"[INFO] Rasterization results cached successfully", flush=True) | |
| # print(f"[DEBUG] example_init completed successfully", flush=True) | |
| def decode(self, codedict, flame, feature_flame, target_rotation=None): | |
| shape_code = codedict[1]['flame_params'].shape.detach() | |
| expr_code = codedict[1]['flame_params'].expression.detach() | |
| # jaw_pose = codedict['jaw_pose'].detach() | |
| # neck_pose = codedict['neck_pose'].detach() | |
| # eyes_pose = codedict['eyes_pose'].detach() | |
| # person_code = codedict['person_code'].detach() | |
| batch_size = shape_code.shape[0] | |
| # condition = torch.cat((expr_code, jaw_pose, eyes_pose, neck_pose), dim=1) | |
| # condition = expr_code.cuda() | |
| condition = torch.cat((expr_code.cuda(), feature_flame), dim=1) | |
| # MLP | |
| condition = condition.unsqueeze(1).repeat(1, self.v_num, 1) | |
| uv_vertices_shape_embeded_condition = torch.cat((self.uv_vertices_shape_embeded, condition), dim=2) | |
| deforms = self.deformNet(uv_vertices_shape_embeded_condition) | |
| deforms = torch.tanh(deforms) | |
| uv_vertices_deforms = 2.0 * deforms[..., :3] | |
| rot_delta_0 = deforms[..., 3:7] | |
| rot_delta_r = torch.exp(rot_delta_0[..., 0]).unsqueeze(-1) | |
| rot_delta_v = rot_delta_0[..., 1:] | |
| rot_delta = torch.cat((rot_delta_r, rot_delta_v), dim=-1) | |
| scale_coef = deforms[..., 7:] | |
| scale_coef = torch.exp(scale_coef) | |
| geometry = flame.forward(codedict[1]['flame_params'], zero_rot=True).cuda() | |
| face_vertices = face_vertices_gen(geometry, self.tri_faces.expand(batch_size, -1, -1)) | |
| # rasterize face_vertices to uv space | |
| D = face_vertices.shape[-1] # 3 | |
| attributes = face_vertices.clone() | |
| attributes = attributes.view(attributes.shape[0] * attributes.shape[1], 3, attributes.shape[-1]) | |
| N, H, W, K, _ = self.bary_coords.shape | |
| idx = self.pix_to_face_ori.clone().view(N * H * W * K, 1, 1).expand(N * H * W * K, 3, D) | |
| pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D) | |
| pixel_vals = (self.bary_coords[..., None] * pixel_face_vals).sum(dim=-2) | |
| uv_vertices = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2) | |
| uv_vertices_flaten = uv_vertices[0].view(uv_vertices.shape[1], -1).permute(1, 0) # batch=1 | |
| uv_vertices = uv_vertices_flaten[self.uvmask_flaten_idx].unsqueeze(0) | |
| # 获取z坐标 | |
| x_coords = uv_vertices[:,:,0] | |
| # y_coords = uv_vertices[:,:,1] | |
| z_coords = uv_vertices[:,:,2] | |
| tan_xy = torch.abs(x_coords)/torch.abs((z_coords+0.01))#0.2 | |
| # 找到z坐标小于0的点的索引 | |
| indices = torch.where((z_coords >= -0.01) | (tan_xy >= 1.732)) | |
| verts_final = uv_vertices + uv_vertices_deforms | |
| # conduct mask | |
| verts_final = verts_final[:, self.uv_head_idx, :] | |
| rot_delta = rot_delta[:, self.uv_head_idx, :] | |
| scale_coef = scale_coef[:, self.uv_head_idx, :] | |
| verts_flame = uv_vertices[:, self.uv_head_idx, :] | |
| # # 根据索引提取满足条件的点 | |
| # verts_final = verts_final[:, indices[1], :] | |
| # verts_flame = verts_flame[:, indices[1], :] | |
| # rotation_mat = rot_mat_from_6dof(codedict[1]['flame_params'].rotation).type(verts_final.dtype).cuda() | |
| # vertices = torch.matmul(rotation_mat.unsqueeze(1), verts_final.unsqueeze(-1)) | |
| # vertices = vertices[..., 0] | |
| scale = torch.clamp(codedict[1]['flame_params'].scale[:, None] + 1.0, 1e-8).cuda().squeeze() | |
| # vertices *= scale # [B, 1, 1] | |
| # translation = codedict[1]['flame_params'].translation.cuda() | |
| # translation[..., 2] = 0.0 | |
| # vertices += translation[:, None] # [B, 1, 3] | |
| # vertices[..., 2] += 2183.5264 | |
| vertices = scale * verts_final | |
| # 计算偏移量的L2损失,使形变保持在合理范围 | |
| lambda_l2 = 0.01 # 降低L2正则化系数,让形变有更大的自由度 | |
| l2_loss = lambda_l2 * torch.norm(uv_vertices_deforms, p=2) # L2范数 | |
| return vertices, rot_delta, scale_coef, verts_flame, verts_final, l2_loss, scale | |
| def decode_single(self, codedict, flame, feature_flame): | |
| shape_code = codedict.shape.detach() | |
| expr_code = codedict.expression.detach() | |
| # jaw_pose = codedict['jaw_pose'].detach() | |
| # neck_pose = codedict['neck_pose'].detach() | |
| # eyes_pose = codedict['eyes_pose'].detach() | |
| # person_code = codedict['person_code'].detach() | |
| batch_size = shape_code.shape[0] | |
| # condition = torch.cat((expr_code, jaw_pose, eyes_pose, neck_pose), dim=1) | |
| # condition = expr_code.cuda() | |
| condition = torch.cat((expr_code.cuda(), feature_flame), dim=1) | |
| # MLP | |
| condition = condition.unsqueeze(1).repeat(1, self.v_num, 1) | |
| uv_vertices_shape_embeded_condition = torch.cat((self.uv_vertices_shape_embeded, condition), dim=2) | |
| deforms = self.deformNet(uv_vertices_shape_embeded_condition) | |
| deforms = torch.tanh(deforms) | |
| uv_vertices_deforms = deforms[..., :3] | |
| rot_delta_0 = deforms[..., 3:7] | |
| rot_delta_r = torch.exp(rot_delta_0[..., 0]).unsqueeze(-1) | |
| rot_delta_v = rot_delta_0[..., 1:] | |
| rot_delta = torch.cat((rot_delta_r, rot_delta_v), dim=-1) | |
| scale_coef = deforms[..., 7:] | |
| scale_coef = torch.exp(scale_coef) | |
| geometry = flame.forward(codedict, zero_rot=True).cuda() | |
| face_vertices = face_vertices_gen(geometry, self.tri_faces.expand(batch_size, -1, -1)) | |
| # rasterize face_vertices to uv space | |
| D = face_vertices.shape[-1] # 3 | |
| attributes = face_vertices.clone() | |
| attributes = attributes.view(attributes.shape[0] * attributes.shape[1], 3, attributes.shape[-1]) | |
| N, H, W, K, _ = self.bary_coords.shape | |
| idx = self.pix_to_face_ori.clone().view(N * H * W * K, 1, 1).expand(N * H * W * K, 3, D) | |
| pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D) | |
| pixel_vals = (self.bary_coords[..., None] * pixel_face_vals).sum(dim=-2) | |
| uv_vertices = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2) | |
| uv_vertices_flaten = uv_vertices[0].view(uv_vertices.shape[1], -1).permute(1, 0) # batch=1 | |
| uv_vertices = uv_vertices_flaten[self.uvmask_flaten_idx].unsqueeze(0) | |
| # 获取z坐标 | |
| x_coords = uv_vertices[:,:,0] | |
| # y_coords = uv_vertices[:,:,1] | |
| z_coords = uv_vertices[:,:,2] | |
| tan_xy = torch.abs(x_coords)/torch.abs((z_coords+0.01))#0.2 | |
| # 找到z坐标小于0的点的索引 | |
| indices = torch.where((z_coords >= -0.01) | (tan_xy >= 1.732)) | |
| verts_final = uv_vertices + uv_vertices_deforms | |
| # conduct mask | |
| verts_final = verts_final[:, self.uv_head_idx, :] | |
| rot_delta = rot_delta[:, self.uv_head_idx, :] | |
| scale_coef = scale_coef[:, self.uv_head_idx, :] | |
| verts_flame = uv_vertices[:, self.uv_head_idx, :] | |
| # # 根据索引提取满足条件的点 | |
| # verts_final = verts_final[:, indices[1], :] | |
| # verts_flame = verts_flame[:, indices[1], :] | |
| # rotation_mat = rot_mat_from_6dof(codedict[1]['flame_params'].rotation).type(verts_final.dtype).cuda() | |
| # vertices = torch.matmul(rotation_mat.unsqueeze(1), verts_final.unsqueeze(-1)) | |
| # vertices = vertices[..., 0] | |
| scale = torch.clamp(codedict.scale[:, None] + 1.0, 1e-8).cuda().squeeze() | |
| # vertices *= scale # [B, 1, 1] | |
| # translation = codedict[1]['flame_params'].translation.cuda() | |
| # translation[..., 2] = 0.0 | |
| # vertices += translation[:, None] # [B, 1, 3] | |
| # vertices[..., 2] += 2183.5264 | |
| vertices = scale * verts_final | |
| # 计算偏移量的L2损失,使形变保持在合理范围 | |
| lambda_l2 = 0.03 # 降低L2正则化系数,让形变有更大的自由度 | |
| l2_loss = lambda_l2 * torch.norm(uv_vertices_deforms, p=2) # L2范数 | |
| return vertices, rot_delta, scale_coef, verts_flame, verts_final, l2_loss, scale | |
| def decode_single3(self, codedict, flame, feature_flame): | |
| shape_code = codedict.shape.detach() | |
| expr_code = codedict.expression.detach() | |
| batch_size = shape_code.shape[0] | |
| # 表情编码 + 特征 | |
| condition = torch.cat((expr_code.cuda(), feature_flame), dim=1) | |
| condition = condition.unsqueeze(1).repeat(1, self.v_num, 1) | |
| uv_vertices_shape_embeded_condition = torch.cat((self.uv_vertices_shape_embeded, condition), dim=2) | |
| # MLP 解码器 | |
| deforms = self.deformNet(uv_vertices_shape_embeded_condition) | |
| deforms = torch.tanh(deforms) | |
| # 只使用偏移信息 | |
| uv_vertices_deforms = deforms[..., :3] | |
| # 获取原始 flame 网格 | |
| geometry = flame.forward(codedict, zero_rot=True).cuda() | |
| face_vertices = face_vertices_gen(geometry, self.tri_faces.expand(batch_size, -1, -1)) | |
| # rasterize 到 UV 空间 | |
| D = face_vertices.shape[-1] | |
| attributes = face_vertices.view(-1, 3, D) | |
| N, H, W, K, _ = self.bary_coords.shape | |
| idx = self.pix_to_face_ori.clone().view(N * H * W * K, 1, 1).expand(-1, 3, D) | |
| pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D) | |
| pixel_vals = (self.bary_coords[..., None] * pixel_face_vals).sum(dim=-2) | |
| uv_vertices = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2) | |
| uv_vertices_flaten = uv_vertices[0].view(uv_vertices.shape[1], -1).permute(1, 0) | |
| uv_vertices = uv_vertices_flaten[self.uvmask_flaten_idx].unsqueeze(0) | |
| # 形变 + mask 头部区域 | |
| verts_final = uv_vertices + uv_vertices_deforms | |
| # 计算缩放并应用(z 方向不加) | |
| scale = torch.clamp(codedict.scale[:, None] + 1.0, 1e-8).cuda().squeeze() | |
| vertices = scale * verts_final | |
| return vertices, verts_final, scale | |
| def decode_single2(self, codedict, flame, feature_flame): | |
| # 总体计时开始 | |
| # torch.cuda.synchronize() | |
| # total_start_time = time.time() | |
| shape_code = codedict.shape.detach() | |
| expr_code = codedict.expression.detach() | |
| batch_size = shape_code.shape[0] | |
| # 步骤1: 表情编码和特征处理 | |
| # torch.cuda.synchronize() | |
| # step1_start = time.time() | |
| condition = torch.cat((expr_code.cuda(), feature_flame), dim=1) | |
| condition = condition.unsqueeze(1).repeat(1, self.v_num, 1) | |
| uv_vertices_shape_embeded_condition = torch.cat((self.uv_vertices_shape_embeded, condition), dim=2) | |
| # torch.cuda.synchronize() | |
| # step1_time = time.time() - step1_start | |
| # print(f"[TIMING] decode_single2 Step 1 - Feature processing: {step1_time:.4f} seconds", flush=True) | |
| # # 步骤2: MLP解码器 | |
| # torch.cuda.synchronize() | |
| # step2_start = time.time() | |
| deforms = self.deformNet(uv_vertices_shape_embeded_condition) | |
| deforms = torch.tanh(deforms) | |
| uv_vertices_deforms = deforms[..., :3] | |
| # torch.cuda.synchronize() | |
| # step2_time = time.time() - step2_start | |
| # print(f"[TIMING] decode_single2 Step 2 - MLP inference: {step2_time:.4f} seconds", flush=True) | |
| # # 步骤3: FLAME几何计算(带缓存) | |
| # torch.cuda.synchronize() | |
| # step3_start = time.time() | |
| if not hasattr(self, 'cached_flame_geometry') or self.cached_flame_geometry is None: | |
| # print(f"[INFO] Computing FLAME geometry (first time)", flush=True) | |
| geometry = flame.forward(codedict, zero_rot=True).cuda() | |
| # 缓存FLAME几何结果 | |
| self.cached_flame_geometry = geometry | |
| self.cached_flame_params = { | |
| 'shape': codedict.shape.clone(), | |
| 'expression': codedict.expression.clone() | |
| } | |
| else: | |
| # 检查参数是否变化 | |
| # 在缓存参数时确保它们在GPU上 | |
| self.cached_flame_params['expression'] = self.cached_flame_params['expression'].to(codedict.expression.device) | |
| shape_changed = torch.norm(codedict.shape - self.cached_flame_params['shape']) > 1e-6 | |
| expr_changed = torch.norm(codedict.expression - self.cached_flame_params['expression']) > 1e-6 | |
| if shape_changed or expr_changed: | |
| # print(f"[INFO] FLAME parameters changed, recomputing geometry", flush=True) | |
| geometry = flame.forward(codedict, zero_rot=True).cuda() | |
| self.cached_flame_geometry = geometry | |
| self.cached_flame_params['shape'] = codedict.shape.clone() | |
| self.cached_flame_params['expression'] = codedict.expression.clone() | |
| else: | |
| # print(f"[INFO] Using cached FLAME geometry", flush=True) | |
| geometry = self.cached_flame_geometry | |
| face_vertices = face_vertices_gen(geometry, self.tri_faces.expand(batch_size, -1, -1)) | |
| # torch.cuda.synchronize() | |
| # step3_time = time.time() - step3_start | |
| # print(f"[TIMING] decode_single2 Step 3 - FLAME geometry: {step3_time:.4f} seconds", flush=True) | |
| # 步骤4: UV处理(带缓存优化) | |
| # torch.cuda.synchronize() | |
| # step4_start = time.time() | |
| if hasattr(self, 'uv_vertices_shape') and self.uv_vertices_shape is not None: | |
| # print(f"[INFO] Using cached UV vertices from example_init", flush=True) | |
| uv_vertices = self.uv_vertices_shape | |
| else: | |
| # print(f"[INFO] Computing UV rasterization (fallback)", flush=True) | |
| # rasterize 到 UV 空间 | |
| D = face_vertices.shape[-1] | |
| attributes = face_vertices.view(-1, 3, D) | |
| N, H, W, K, _ = self.bary_coords.shape | |
| idx = self.pix_to_face_ori.clone().view(N * H * W * K, 1, 1).expand(-1, 3, D) | |
| pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D) | |
| pixel_vals = (self.bary_coords[..., None] * pixel_face_vals).sum(dim=-2) | |
| uv_vertices = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2) | |
| uv_vertices_flaten = uv_vertices[0].view(uv_vertices.shape[1], -1).permute(1, 0) | |
| uv_vertices = uv_vertices_flaten[self.uvmask_flaten_idx].unsqueeze(0) | |
| # torch.cuda.synchronize() | |
| # step4_time = time.time() - step4_start | |
| # print(f"[TIMING] decode_single2 Step 4 - UV processing: {step4_time:.4f} seconds", flush=True) | |
| # # 步骤5: 最终计算 | |
| # torch.cuda.synchronize() | |
| # step5_start = time.time() | |
| # 形变 + mask 头部区域 | |
| verts_final = uv_vertices + uv_vertices_deforms | |
| # 计算缩放并应用(z 方向不加) | |
| scale = torch.clamp(codedict.scale[:, None] + 1.0, 1e-8).cuda().squeeze() | |
| vertices = scale * verts_final | |
| torch.cuda.synchronize() | |
| # step5_time = time.time() - step5_start | |
| # print(f"[TIMING] decode_single2 Step 5 - Final computation: {step5_time:.4f} seconds", flush=True) | |
| # # 总体计时结束 | |
| # torch.cuda.synchronize() | |
| # total_time = time.time() - total_start_time | |
| # print(f"[TIMING] decode_single2 Total time: {total_time:.4f} seconds", flush=True) | |
| # print(f"[TIMING] decode_single2 Breakdown: Features({step1_time:.4f}s) + MLP({step2_time:.4f}s) + FLAME({step3_time:.4f}s) + UV({step4_time:.4f}s) + Final({step5_time:.4f}s)", flush=True) | |
| return vertices, verts_final, scale | |
| def capture(self): | |
| return ( | |
| self.deformNet.state_dict(), | |
| self.optimizer.state_dict(), | |
| ) | |
| def restore(self, model_args): | |
| (net_dict, | |
| opt_dict) = model_args | |
| self.deformNet.load_state_dict(net_dict) | |
| self.training_setup() | |
| self.optimizer.load_state_dict(opt_dict) | |
| def training_setup(self): | |
| params_group = [ | |
| {'params': self.deformNet.parameters(), 'lr': 1e-4}, | |
| ] | |
| self.optimizer = torch.optim.Adam(params_group, betas=(0.9, 0.999)) | |
| def get_template(self): | |
| geometry_template = self.flame_model.forward_geo( | |
| self.default_shape_code, | |
| self.default_expr_code, | |
| self.default_jaw_pose, | |
| eye_pose_params=self.default_eyes_pose, | |
| ) | |
| return geometry_template | |
| class MLP(nn.Module): | |
| def __init__(self, input_dim, output_dim, hidden_dim=256, hidden_layers=8): | |
| super().__init__() | |
| self.hidden_dim = hidden_dim | |
| self.hidden_layers = hidden_layers | |
| self.input_dim = input_dim | |
| self.output_dim = output_dim | |
| self.fcs = nn.ModuleList( | |
| [nn.Linear(input_dim, hidden_dim)] + [nn.Linear(hidden_dim, hidden_dim) for i in range(hidden_layers-1)] | |
| ) | |
| self.output_linear = nn.Linear(hidden_dim, output_dim) | |
| def forward(self, input): | |
| # input: B,V,d | |
| batch_size, N_v, input_dim = input.shape | |
| input_ori = input.reshape(batch_size*N_v, -1) | |
| h = input_ori | |
| for i, l in enumerate(self.fcs): | |
| h = self.fcs[i](h) | |
| h = F.relu(h) | |
| output = self.output_linear(h) | |
| output = output.reshape(batch_size, N_v, -1) | |
| return output | |
| class MLP2(nn.Module): | |
| def __init__(self, input_dim, condition_dim, output_dim1, output_dim2, hidden_dim=256, hidden_layers=8): | |
| super().__init__() | |
| self.hidden_dim = hidden_dim | |
| self.condition_dim = condition_dim | |
| self.hidden_layers = hidden_layers | |
| mid_layers = math.ceil(hidden_layers/2.) | |
| self.input_dim = input_dim | |
| self.output_dim1 = output_dim1 | |
| self.output_dim2 = output_dim2 | |
| self.fcs1 = nn.ModuleList( | |
| [nn.Linear(input_dim, hidden_dim)] + [nn.Linear(hidden_dim, hidden_dim) if i!=(mid_layers-1) else nn.Linear(hidden_dim, hidden_dim+output_dim1) for i in range(mid_layers)] | |
| ) | |
| self.fcs2 = nn.ModuleList( | |
| [nn.Linear(hidden_dim, hidden_dim) if i!=0 else nn.Linear(hidden_dim+condition_dim, hidden_dim) for i in range(mid_layers, hidden_layers-1)] | |
| ) | |
| self.output_linear = nn.Linear(hidden_dim, output_dim2) | |
| def forward(self, input, condition): | |
| # input: B,V,d1 | |
| # condition: B,d2 | |
| batch_size, N_v, input_dim = input.shape | |
| input_ori = input.reshape(batch_size*N_v, -1) | |
| h = input_ori | |
| for i, l in enumerate(self.fcs1): | |
| h = self.fcs1[i](h) | |
| h = F.relu(h) | |
| oup1 = h[:, -self.output_dim1:] | |
| h = h[:, :-self.output_dim1] | |
| ... | |
| for i, l in enumerate(self.fcs2): | |
| h = self.fcs1[i](h) | |
| h = F.relu(h) | |
| # input_ori = input.reshape(batch_size*N_v, -1) | |
| # h = input_ori | |
| # for i, l in enumerate(self.fcs): | |
| # h = self.fcs[i](h) | |
| # h = F.relu(h) | |
| # output = self.output_linear(h) | |
| # output = output.reshape(batch_size, N_v, -1) | |
| class SIRENMLP(nn.Module): | |
| def __init__(self, | |
| input_dim=3, | |
| output_dim=3, | |
| hidden_dim=256, | |
| hidden_layers=8, | |
| condition_dim=100, | |
| device=None): | |
| super().__init__() | |
| self.device = device | |
| self.input_dim = input_dim | |
| self.z_dim = condition_dim | |
| self.hidden_layers = hidden_layers | |
| self.hidden_dim = hidden_dim | |
| self.output_dim = output_dim | |
| self.network = nn.ModuleList( | |
| [FiLMLayer(self.input_dim, self.hidden_dim)] + | |
| [FiLMLayer(self.hidden_dim, self.hidden_dim) for i in range(self.hidden_layers - 1)] | |
| ) | |
| self.final_layer = nn.Linear(self.hidden_dim, self.output_dim) | |
| self.mapping_network = MappingNetwork(condition_dim, 256, | |
| len(self.network) * self.hidden_dim * 2) | |
| self.network.apply(frequency_init(25)) | |
| # self.final_layer.apply(frequency_init(25)) | |
| self.final_layer.weight.data.normal_(0.0, 0.) | |
| self.final_layer.bias.data.fill_(0.) | |
| self.network[0].apply(first_layer_film_sine_init) | |
| def forward_vector(self, input, z): | |
| frequencies, phase_shifts = self.mapping_network(z) | |
| return self.forward_with_frequencies_phase_shifts(input, frequencies, phase_shifts) | |
| def forward_with_frequencies_phase_shifts(self, input, frequencies, phase_shifts): | |
| frequencies = frequencies * 15 + 30 | |
| x = input | |
| for index, layer in enumerate(self.network): | |
| start = index * self.hidden_dim | |
| end = (index + 1) * self.hidden_dim | |
| x = layer(x, frequencies[..., start:end], phase_shifts[..., start:end]) | |
| sigma = self.final_layer(x) | |
| return sigma | |
| def forward(self, vertices, additional_conditioning): | |
| # vertices: in canonical space of flame | |
| # vertex eval: torch.Size([N, V, 3]) | |
| # map eval: torch.Size([N, 3, H, W]) | |
| # conditioning | |
| # torch.Size([N, C]) | |
| # vertex inputs (N, V, 3) -> (N, 3, V, 1) | |
| vertices = vertices.permute(0, 2, 1)[:, :, :, None] | |
| b, c, h, w = vertices.shape | |
| # to vector | |
| x = vertices.permute(0, 2, 3, 1).reshape(b, -1, c) | |
| z = additional_conditioning # .repeat(1, h*w, 1) | |
| # apply siren network | |
| o = self.forward_vector(x, z) | |
| # to image | |
| o = o.reshape(b, h, w, self.output_dim) | |
| output = o.permute(0, 3, 1, 2) | |
| return output[:, :, :, 0].permute(0, 2, 1) | |
| class FiLMLayer(nn.Module): | |
| def __init__(self, input_dim, hidden_dim): | |
| super().__init__() | |
| self.layer = nn.Linear(input_dim, hidden_dim) | |
| def forward(self, x, freq, phase_shift, ignore_conditions=None): | |
| x = self.layer(x) | |
| if ignore_conditions is not None: | |
| cond_freq, cond_phase_shift = freq[:-1], phase_shift[:-1] | |
| cond_freq = cond_freq.unsqueeze(1).expand_as(x).clone() | |
| cond_phase_shift = cond_phase_shift.unsqueeze(1).expand_as(x).clone() | |
| ignore_freq, ignore_phase_shift = freq[-1:], phase_shift[-1:] | |
| ignore_freq = ignore_freq.unsqueeze(1).expand_as(x) | |
| ignore_phase_shift = ignore_phase_shift.unsqueeze(1).expand_as(x) | |
| cond_freq[:, ignore_conditions] = ignore_freq[:, ignore_conditions] | |
| cond_phase_shift[:, ignore_conditions] = ignore_phase_shift[:, ignore_conditions] | |
| freq, phase_shift = cond_freq, cond_phase_shift | |
| else: | |
| freq = freq.unsqueeze(1).expand_as(x) | |
| phase_shift = phase_shift.unsqueeze(1).expand_as(x) | |
| # print('x', x.shape) | |
| # print('freq', freq.shape) | |
| # print('phase_shift', phase_shift.shape) | |
| # x torch.Size([6, 5023, 256]) | |
| # freq torch.Size([6, 5023, 256]) | |
| # phase_shift torch.Size([6, 5023, 256]) | |
| return torch.sin(freq * x + phase_shift) | |
| def frequency_init(freq): | |
| def init(m): | |
| with torch.no_grad(): | |
| if isinstance(m, nn.Linear): | |
| num_input = m.weight.size(-1) | |
| m.weight.uniform_(-np.sqrt(6 / num_input) / freq, np.sqrt(6 / num_input) / freq) | |
| elif isinstance(m, nn.Conv2d): | |
| num_input = torch.prod( | |
| torch.tensor(m.weight.shape[1:], device=m.weight.device)).cpu().item() | |
| m.weight.uniform_(-np.sqrt(6 / num_input) / freq, np.sqrt(6 / num_input) / freq) | |
| return init | |
| class MappingNetwork(nn.Module): | |
| def __init__(self, z_dim, map_hidden_dim, map_output_dim): | |
| super().__init__() | |
| self.network = nn.Sequential(nn.Linear(z_dim, map_hidden_dim), | |
| nn.LeakyReLU(0.2, inplace=True), | |
| nn.Linear(map_hidden_dim, map_hidden_dim), | |
| nn.LeakyReLU(0.2, inplace=True), | |
| nn.Linear(map_hidden_dim, map_hidden_dim), | |
| nn.LeakyReLU(0.2, inplace=True), | |
| nn.Linear(map_hidden_dim, map_output_dim)) | |
| self.network.apply(kaiming_leaky_init) | |
| with torch.no_grad(): | |
| self.network[-1].weight *= 0.25 | |
| def forward(self, z): | |
| frequencies_offsets = self.network(z) | |
| frequencies = frequencies_offsets[..., :frequencies_offsets.shape[-1] // 2] | |
| phase_shifts = frequencies_offsets[..., frequencies_offsets.shape[-1] // 2:] | |
| return frequencies, phase_shifts | |
| def kaiming_leaky_init(m): | |
| classname = m.__class__.__name__ | |
| if classname.find('Linear') != -1: | |
| torch.nn.init.kaiming_normal_(m.weight, a=0.2, mode='fan_in', nonlinearity='leaky_relu') | |
| def first_layer_film_sine_init(m): | |
| with torch.no_grad(): | |
| if isinstance(m, nn.Linear): | |
| num_input = m.weight.size(-1) | |
| m.weight.uniform_(-1 / num_input, 1 / num_input) | |
| elif isinstance(m, nn.Conv2d): | |
| num_input = torch.prod( | |
| torch.tensor(m.weight.shape[1:], device=m.weight.device)).cpu().item() | |
| m.weight.uniform_(-1 / num_input, 1 / num_input) | |